Nodes.py 353 KB
Newer Older
William Stein's avatar
William Stein committed
1
#
2
#   Parse tree nodes
William Stein's avatar
William Stein committed
3
#
4

5 6
from __future__ import absolute_import

7
import cython
8
cython.declare(sys=object, os=object, copy=object,
9
               Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object,
10
               py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object,
11
               StructOrUnionScope=object, PyClassScope=object,
12
               CppClassScope=object, UtilityCode=object, EncodedString=object,
13
               error_type=object, _py_int_types=object)
William Stein's avatar
William Stein committed
14

15
import sys, os, copy
Stefan Behnel's avatar
Stefan Behnel committed
16
from itertools import chain
Robert Bradshaw's avatar
Robert Bradshaw committed
17

18 19 20 21 22 23 24
from . import Builtin
from .Errors import error, warning, InternalError, CompileError
from . import Naming
from . import PyrexTypes
from . import TypeSlots
from .PyrexTypes import py_object_type, error_type
from .Symtab import (ModuleScope, LocalScope, ClosureScope,
25
                     StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope)
26
from .Code import UtilityCode
27
from .StringEncoding import EncodedString
28
from . import Future
29 30
from . import Options
from . import DebugFlags
31
from .Pythran import has_np_pythran, pythran_type, is_pythran_buffer
32
from ..Utils import add_metaclass
William Stein's avatar
William Stein committed
33

34

35 36 37 38 39
if sys.version_info[0] >= 3:
    _py_int_types = int
else:
    _py_int_types = (int, long)

40

41
def relative_position(pos):
42
    return (pos[0].get_filenametable_entry(), pos[1])
43

44 45 46 47

def embed_position(pos, docstring):
    if not Options.embed_pos_in_docstring:
        return docstring
48
    pos_line = u'File: %s (starting at line %s)' % relative_position(pos)
49 50
    if docstring is None:
        # unicode string
51
        return EncodedString(pos_line)
52 53 54 55 56 57

    # make sure we can encode the filename in the docstring encoding
    # otherwise make the docstring a unicode string
    encoding = docstring.encoding
    if encoding is not None:
        try:
Stefan Behnel's avatar
Stefan Behnel committed
58
            pos_line.encode(encoding)
59 60 61 62 63
        except UnicodeEncodeError:
            encoding = None

    if not docstring:
        # reuse the string encoding of the original docstring
64
        doc = EncodedString(pos_line)
65
    else:
66
        doc = EncodedString(pos_line + u'\n' + docstring)
67 68
    doc.encoding = encoding
    return doc
69

70

71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
def _analyse_signature_annotation(annotation, env):
    base_type = None
    explicit_pytype = explicit_ctype = False
    if annotation.is_dict_literal:
        for name, value in annotation.key_value_pairs:
            if not name.is_string_literal:
                continue
            if name.value in ('type', b'type'):
                explicit_pytype = True
                if not explicit_ctype:
                    annotation = value
            elif name.value in ('ctype', b'ctype'):
                explicit_ctype = True
                annotation = value
        if explicit_pytype and explicit_ctype:
            warning(annotation.pos, "Duplicate type declarations found in signature annotation")
    arg_type = annotation.analyse_as_type(env)
    if arg_type is not None:
        if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject:
            warning(annotation.pos,
                    "Python type declaration in signature annotation does not refer to a Python type")
        base_type = CAnalysedBaseTypeNode(
            annotation.pos, type=arg_type, is_arg=True)
    else:
        warning(annotation.pos, "Unknown type declaration found in signature annotation")
    return base_type, arg_type


99
def write_func_call(func, codewriter_class):
100
    def f(*args, **kwds):
101
        if len(args) > 1 and isinstance(args[1], codewriter_class):
Robert Bradshaw's avatar
Robert Bradshaw committed
102 103
            # here we annotate the code with this function call
            # but only if new code is generated
104
            node, code = args[:2]
Robert Bradshaw's avatar
Robert Bradshaw committed
105
            marker = '                    /* %s -> %s.%s %s */' % (
Stefan Behnel's avatar
Stefan Behnel committed
106 107 108 109
                ' ' * code.call_level,
                node.__class__.__name__,
                func.__name__,
                node.pos[1:])
Robert Bradshaw's avatar
Robert Bradshaw committed
110 111 112 113 114 115 116
            pristine = code.buffer.stream.tell()
            code.putln(marker)
            start = code.buffer.stream.tell()
            code.call_level += 4
            res = func(*args, **kwds)
            code.call_level -= 4
            if start == code.buffer.stream.tell():
117 118
                # no code written => undo writing marker
                code.buffer.stream.truncate(pristine)
Robert Bradshaw's avatar
Robert Bradshaw committed
119
            else:
120
                marker = marker.replace('->', '<-', 1)
Robert Bradshaw's avatar
Robert Bradshaw committed
121 122 123 124
                code.putln(marker)
            return res
        else:
            return func(*args, **kwds)
125 126
    return f

127

128 129
class VerboseCodeWriter(type):
    # Set this as a metaclass to trace function calls in code.
130
    # This slows down code generation and makes much larger files.
131
    def __new__(cls, name, bases, attrs):
132
        from types import FunctionType
133
        from .Code import CCodeWriter
134 135 136
        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType):
137
                attrs[mname] = write_func_call(m, CCodeWriter)
138 139 140
        return super(VerboseCodeWriter, cls).__new__(cls, name, bases, attrs)


141 142 143 144 145 146 147 148 149 150 151 152 153
class CheckAnalysers(type):
    """Metaclass to check that type analysis functions return a node.
    """
    methods = set(['analyse_types',
                   'analyse_expressions',
                   'analyse_target_types'])

    def __new__(cls, name, bases, attrs):
        from types import FunctionType
        def check(name, func):
            def call(*args, **kwargs):
                retval = func(*args, **kwargs)
                if retval is None:
154
                    print('%s %s %s' % (name, args, kwargs))
155 156 157 158 159 160 161 162 163 164
                return retval
            return call

        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType) and mname in cls.methods:
                attrs[mname] = check(mname, m)
        return super(CheckAnalysers, cls).__new__(cls, name, bases, attrs)


165 166 167 168 169 170 171 172
def _with_metaclass(cls):
    if DebugFlags.debug_trace_code_generation:
        return add_metaclass(VerboseCodeWriter)(cls)
    #return add_metaclass(CheckAnalysers)(cls)
    return cls


@_with_metaclass
Stefan Behnel's avatar
Stefan Behnel committed
173
class Node(object):
William Stein's avatar
William Stein committed
174 175 176
    #  pos         (string, int, int)   Source file position
    #  is_name     boolean              Is a NameNode
    #  is_literal  boolean              Is a ConstNode
177

William Stein's avatar
William Stein committed
178
    is_name = 0
179
    is_none = 0
180
    is_nonecheck = 0
William Stein's avatar
William Stein committed
181
    is_literal = 0
182
    is_terminator = 0
183
    is_wrapper = False  # is a DefNode wrapper for a C function
184
    temps = None
185

Stefan Behnel's avatar
Stefan Behnel committed
186
    # All descendants should set child_attrs to a list of the attributes
187 188
    # containing nodes considered "children" in the tree. Each such attribute
    # can either contain a single node or a list of nodes. See Visitor.py.
189
    child_attrs = None
190

191 192
    cf_state = None

193 194 195 196 197
    # This may be an additional (or 'actual') type that will be checked when
    # this node is coerced to another type. This could be useful to set when
    # the actual type to which it can coerce is known, but you want to leave
    # the type a py_object_type
    coercion_type = None
198

William Stein's avatar
William Stein committed
199 200 201
    def __init__(self, pos, **kw):
        self.pos = pos
        self.__dict__.update(kw)
202

203 204
    gil_message = "Operation"

205
    nogil_check = None
206

207
    def gil_error(self, env=None):
208
        error(self.pos, "%s not allowed without gil" % self.gil_message)
209

Robert Bradshaw's avatar
Robert Bradshaw committed
210
    cpp_message = "Operation"
211

Robert Bradshaw's avatar
Robert Bradshaw committed
212 213 214 215 216 217
    def cpp_check(self, env):
        if not env.is_cpp():
            self.cpp_error()

    def cpp_error(self):
        error(self.pos, "%s only allowed in c++" % self.cpp_message)
218

219 220 221 222
    def clone_node(self):
        """Clone the node. This is defined as a shallow copy, except for member lists
           amongst the child attributes (from get_child_accessors) which are also
           copied. Lists containing child nodes are thus seen as a way for the node
Stefan Behnel's avatar
Stefan Behnel committed
223
           to hold multiple children directly; the list is not treated as a separate
224
           level in the tree."""
225 226 227
        result = copy.copy(self)
        for attrname in result.child_attrs:
            value = getattr(result, attrname)
228
            if isinstance(value, list):
229
                setattr(result, attrname, [x for x in value])
230
        return result
231 232


William Stein's avatar
William Stein committed
233
    #
234
    #  There are 3 phases of parse tree processing, applied in order to
William Stein's avatar
William Stein committed
235 236
    #  all the statements in a given scope-block:
    #
237
    #  (0) analyse_declarations
William Stein's avatar
William Stein committed
238 239 240 241
    #        Make symbol table entries for all declarations at the current
    #        level, both explicit (def, cdef, etc.) and implicit (assignment
    #        to an otherwise undeclared name).
    #
242
    #  (1) analyse_expressions
William Stein's avatar
William Stein committed
243 244
    #         Determine the result types of expressions and fill in the
    #         'type' attribute of each ExprNode. Insert coercion nodes into the
245
    #         tree where needed to convert to and from Python objects.
William Stein's avatar
William Stein committed
246 247 248 249
    #         Allocate temporary locals for intermediate results. Fill
    #         in the 'result_code' attribute of each ExprNode with a C code
    #         fragment.
    #
250
    #  (2) generate_code
William Stein's avatar
William Stein committed
251 252 253 254
    #         Emit C code for all declarations, statements and expressions.
    #         Recursively applies the 3 processing phases to the bodies of
    #         functions.
    #
255

William Stein's avatar
William Stein committed
256 257
    def analyse_declarations(self, env):
        pass
258

William Stein's avatar
William Stein committed
259 260 261
    def analyse_expressions(self, env):
        raise InternalError("analyse_expressions not implemented for %s" % \
            self.__class__.__name__)
262

William Stein's avatar
William Stein committed
263 264 265
    def generate_code(self, code):
        raise InternalError("generate_code not implemented for %s" % \
            self.__class__.__name__)
266

267 268 269 270
    def annotate(self, code):
        # mro does the wrong thing
        if isinstance(self, BlockNode):
            self.body.annotate(code)
271

272 273 274 275
    def end_pos(self):
        try:
            return self._end_pos
        except AttributeError:
Stefan Behnel's avatar
Stefan Behnel committed
276
            pos = self.pos
277 278 279
            if not self.child_attrs:
                self._end_pos = pos
                return pos
280
            for attr in self.child_attrs:
281
                child = getattr(self, attr)
282
                # Sometimes lists, sometimes nodes
283 284 285
                if child is None:
                    pass
                elif isinstance(child, list):
Stefan Behnel's avatar
Stefan Behnel committed
286 287
                    for c in child:
                        pos = max(pos, c.end_pos())
288
                else:
Stefan Behnel's avatar
Stefan Behnel committed
289 290 291
                    pos = max(pos, child.end_pos())
            self._end_pos = pos
            return pos
William Stein's avatar
William Stein committed
292

293
    def dump(self, level=0, filter_out=("pos",), cutoff=100, encountered=None):
294 295
        """Debug helper method that returns a recursive string representation of this node.
        """
296 297
        if cutoff == 0:
            return "<...nesting level cutoff...>"
298 299 300
        if encountered is None:
            encountered = set()
        if id(self) in encountered:
301
            return "<%s (0x%x) -- already output>" % (self.__class__.__name__, id(self))
302
        encountered.add(id(self))
303

304 305
        def dump_child(x, level):
            if isinstance(x, Node):
306
                return x.dump(level, filter_out, cutoff-1, encountered)
307
            elif isinstance(x, list):
Robert Bradshaw's avatar
Robert Bradshaw committed
308
                return "[%s]" % ", ".join([dump_child(item, level) for item in x])
309 310
            else:
                return repr(x)
311

312
        attrs = [(key, value) for key, value in self.__dict__.items() if key not in filter_out]
313
        if len(attrs) == 0:
314
            return "<%s (0x%x)>" % (self.__class__.__name__, id(self))
315 316
        else:
            indent = "  " * level
317
            res = "<%s (0x%x)\n" % (self.__class__.__name__, id(self))
318 319 320 321
            for key, value in attrs:
                res += "%s  %s: %s\n" % (indent, key, dump_child(value, level + 1))
            res += "%s>" % indent
            return res
322

323 324 325 326 327 328
    def dump_pos(self, mark_column=False, marker='(#)'):
        """Debug helper method that returns the source code context of this node as a string.
        """
        if not self.pos:
            return u''
        source_desc, line, col = self.pos
329
        contents = source_desc.get_lines(encoding='ASCII', error_handling='ignore')
330
        # line numbers start at 1
331
        lines = contents[max(0, line-3):line]
332 333 334 335 336 337 338 339
        current = lines[-1]
        if mark_column:
            current = current[:col] + marker + current[col:]
        lines[-1] = current.rstrip() + u'             # <<<<<<<<<<<<<<\n'
        lines += contents[line:line+2]
        return u'"%s":%d:%d\n%s\n' % (
            source_desc.get_escaped_description(), line, col, u''.join(lines))

340 341 342 343 344 345 346 347 348 349 350 351 352 353
class CompilerDirectivesNode(Node):
    """
    Sets compiler directives for the children nodes
    """
    #  directives     {string:value}  A dictionary holding the right value for
    #                                 *all* possible directives.
    #  body           Node
    child_attrs = ["body"]

    def analyse_declarations(self, env):
        old = env.directives
        env.directives = self.directives
        self.body.analyse_declarations(env)
        env.directives = old
354

355 356 357
    def analyse_expressions(self, env):
        old = env.directives
        env.directives = self.directives
358
        self.body = self.body.analyse_expressions(env)
359
        env.directives = old
360
        return self
361 362 363 364 365 366 367 368

    def generate_function_definitions(self, env, code):
        env_old = env.directives
        code_old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_function_definitions(env, code)
        env.directives = env_old
        code.globalstate.directives = code_old
369

370 371 372 373 374
    def generate_execution_code(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_execution_code(code)
        code.globalstate.directives = old
375

376 377 378 379 380
    def annotate(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.annotate(code)
        code.globalstate.directives = old
381

Stefan Behnel's avatar
Stefan Behnel committed
382
class BlockNode(object):
William Stein's avatar
William Stein committed
383 384
    #  Mixin class for nodes representing a declaration block.

385
    def generate_cached_builtins_decls(self, env, code):
386
        entries = env.global_scope().undeclared_cached_builtins
387
        for entry in entries:
388
            code.globalstate.add_cached_builtin_decl(entry)
389
        del entries[:]
390 391 392 393

    def generate_lambda_definitions(self, env, code):
        for node in env.lambda_defs:
            node.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
394 395 396

class StatListNode(Node):
    # stats     a list of StatNode
397

398
    child_attrs = ["stats"]
399

400
    @staticmethod
401 402
    def create_analysed(pos, env, *args, **kw):
        node = StatListNode(pos, *args, **kw)
403
        return node  # No node-specific analysis needed
404

William Stein's avatar
William Stein committed
405 406
    def analyse_declarations(self, env):
        #print "StatListNode.analyse_declarations" ###
407
        for stat in self.stats:
William Stein's avatar
William Stein committed
408
            stat.analyse_declarations(env)
409

William Stein's avatar
William Stein committed
410 411
    def analyse_expressions(self, env):
        #print "StatListNode.analyse_expressions" ###
412 413
        self.stats = [stat.analyse_expressions(env)
                      for stat in self.stats]
414
        return self
415

416
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
417 418
        #print "StatListNode.generate_function_definitions" ###
        for stat in self.stats:
419
            stat.generate_function_definitions(env, code)
420

William Stein's avatar
William Stein committed
421 422 423 424 425
    def generate_execution_code(self, code):
        #print "StatListNode.generate_execution_code" ###
        for stat in self.stats:
            code.mark_pos(stat.pos)
            stat.generate_execution_code(code)
426

427 428 429
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)
430

William Stein's avatar
William Stein committed
431 432 433 434 435 436 437 438 439 440 441 442 443

class StatNode(Node):
    #
    #  Code generation for statements is split into the following subphases:
    #
    #  (1) generate_function_definitions
    #        Emit C code for the definitions of any structs,
    #        unions, enums and functions defined in the current
    #        scope-block.
    #
    #  (2) generate_execution_code
    #        Emit C code for executable statements.
    #
444

445
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
446
        pass
447

William Stein's avatar
William Stein committed
448 449 450 451 452 453 454 455
    def generate_execution_code(self, code):
        raise InternalError("generate_execution_code not implemented for %s" % \
            self.__class__.__name__)


class CDefExternNode(StatNode):
    #  include_file   string or None
    #  body           StatNode
456

457
    child_attrs = ["body"]
458

William Stein's avatar
William Stein committed
459 460 461 462 463 464 465
    def analyse_declarations(self, env):
        if self.include_file:
            env.add_include_file(self.include_file)
        old_cinclude_flag = env.in_cinclude
        env.in_cinclude = 1
        self.body.analyse_declarations(env)
        env.in_cinclude = old_cinclude_flag
466

William Stein's avatar
William Stein committed
467
    def analyse_expressions(self, env):
468
        return self
469

William Stein's avatar
William Stein committed
470 471
    def generate_execution_code(self, code):
        pass
472 473 474

    def annotate(self, code):
        self.body.annotate(code)
475

William Stein's avatar
William Stein committed
476 477 478 479 480 481 482 483

class CDeclaratorNode(Node):
    # Part of a C declaration.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #      Returns (name, type) pair where name is the
484
    #      CNameDeclaratorNode of the name being declared
William Stein's avatar
William Stein committed
485 486
    #      and type is the type it is being declared as.
    #
487
    #  calling_convention  string   Calling convention of CFuncDeclaratorNode
488
    #                               for which this is a base
489

490 491
    child_attrs = []

492 493
    calling_convention = ""

494 495 496
    def analyse_templates(self):
        # Only C++ functions have templates.
        return None
William Stein's avatar
William Stein committed
497

498

William Stein's avatar
William Stein committed
499
class CNameDeclaratorNode(CDeclaratorNode):
500
    #  name    string             The Cython name being declared
Robert Bradshaw's avatar
Robert Bradshaw committed
501 502
    #  cname   string or None     C name, if specified
    #  default ExprNode or None   the value assigned on declaration
503

Robert Bradshaw's avatar
Robert Bradshaw committed
504
    child_attrs = ['default']
505

506
    default = None
507

508
    def analyse(self, base_type, env, nonempty=0):
509
        if nonempty and self.name == '':
510
            # May have mistaken the name for the type.
511
            if base_type.is_ptr or base_type.is_array or base_type.is_buffer:
512
                error(self.pos, "Missing argument name")
513 514
            elif base_type.is_void:
                error(self.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
515 516 517
            else:
                self.name = base_type.declaration_code("", for_display=1, pyrex=1)
                base_type = py_object_type
518 519 520 521

        if base_type.is_fused and env.fused_to_specific:
            base_type = base_type.specialize(env.fused_to_specific)

522
        self.type = base_type
William Stein's avatar
William Stein committed
523
        return self, base_type
524

525

William Stein's avatar
William Stein committed
526 527
class CPtrDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode
528

529 530
    child_attrs = ["base"]

531 532 533
    def analyse_templates(self):
        return self.base.analyse_templates()

534
    def analyse(self, base_type, env, nonempty=0):
William Stein's avatar
William Stein committed
535
        if base_type.is_pyobject:
536
            error(self.pos, "Pointer base type cannot be a Python object")
William Stein's avatar
William Stein committed
537
        ptr_type = PyrexTypes.c_ptr_type(base_type)
538 539
        return self.base.analyse(ptr_type, env, nonempty=nonempty)

Danilo Freitas's avatar
Danilo Freitas committed
540 541 542 543 544 545

class CReferenceDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

546 547 548
    def analyse_templates(self):
        return self.base.analyse_templates()

549
    def analyse(self, base_type, env, nonempty=0):
Danilo Freitas's avatar
Danilo Freitas committed
550
        if base_type.is_pyobject:
551
            error(self.pos, "Reference base type cannot be a Python object")
552
        ref_type = PyrexTypes.c_ref_type(base_type)
553 554
        return self.base.analyse(ref_type, env, nonempty=nonempty)

Danilo Freitas's avatar
Danilo Freitas committed
555

William Stein's avatar
William Stein committed
556 557 558
class CArrayDeclaratorNode(CDeclaratorNode):
    # base        CDeclaratorNode
    # dimension   ExprNode
559 560

    child_attrs = ["base", "dimension"]
561

562
    def analyse(self, base_type, env, nonempty=0):
563
        if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction:
564
            from .ExprNodes import TupleNode
Robert Bradshaw's avatar
Robert Bradshaw committed
565 566 567 568 569 570 571
            if isinstance(self.dimension, TupleNode):
                args = self.dimension.args
            else:
                args = self.dimension,
            values = [v.analyse_as_type(env) for v in args]
            if None in values:
                ix = values.index(None)
572 573 574 575
                error(args[ix].pos, "Template parameter not a type")
                base_type = error_type
            else:
                base_type = base_type.specialize_here(self.pos, values)
576
            return self.base.analyse(base_type, env, nonempty=nonempty)
William Stein's avatar
William Stein committed
577
        if self.dimension:
578
            self.dimension = self.dimension.analyse_const_expression(env)
William Stein's avatar
William Stein committed
579 580
            if not self.dimension.type.is_int:
                error(self.dimension.pos, "Array dimension not integer")
581 582 583 584 585 586 587
            size = self.dimension.get_constant_c_result_code()
            if size is not None:
                try:
                    size = int(size)
                except ValueError:
                    # runtime constant?
                    pass
William Stein's avatar
William Stein committed
588 589 590
        else:
            size = None
        if not base_type.is_complete():
591
            error(self.pos, "Array element type '%s' is incomplete" % base_type)
William Stein's avatar
William Stein committed
592
        if base_type.is_pyobject:
593
            error(self.pos, "Array element cannot be a Python object")
594
        if base_type.is_cfunction:
595
            error(self.pos, "Array element cannot be a function")
William Stein's avatar
William Stein committed
596
        array_type = PyrexTypes.c_array_type(base_type, size)
597
        return self.base.analyse(array_type, env, nonempty=nonempty)
William Stein's avatar
William Stein committed
598 599 600 601 602


class CFuncDeclaratorNode(CDeclaratorNode):
    # base             CDeclaratorNode
    # args             [CArgDeclNode]
603
    # templates        [TemplatePlaceholderType]
William Stein's avatar
William Stein committed
604 605 606
    # has_varargs      boolean
    # exception_value  ConstNode
    # exception_check  boolean    True if PyErr_Occurred check needed
607 608
    # nogil            boolean    Can be called without gil
    # with_gil         boolean    Acquire gil around function body
609
    # is_const_method  boolean    Whether this is a const method
610

611 612
    child_attrs = ["base", "args", "exception_value"]

613
    overridable = 0
614
    optional_arg_count = 0
615
    is_const_method = 0
616 617 618 619
    templates = None

    def analyse_templates(self):
        if isinstance(self.base, CArrayDeclaratorNode):
620
            from .ExprNodes import TupleNode, NameNode
621 622 623 624 625 626 627
            template_node = self.base.dimension
            if isinstance(template_node, TupleNode):
                template_nodes = template_node.args
            elif isinstance(template_node, NameNode):
                template_nodes = [template_node]
            else:
                error(template_node.pos, "Template arguments must be a list of names")
628
                return None
629 630 631 632 633 634 635 636 637 638
            self.templates = []
            for template in template_nodes:
                if isinstance(template, NameNode):
                    self.templates.append(PyrexTypes.TemplatePlaceholderType(template.name))
                else:
                    error(template.pos, "Template arguments must be a list of names")
            self.base = self.base.base
            return self.templates
        else:
            return None
William Stein's avatar
William Stein committed
639

640 641 642
    def analyse(self, return_type, env, nonempty=0, directive_locals=None):
        if directive_locals is None:
            directive_locals = {}
643 644
        if nonempty:
            nonempty -= 1
William Stein's avatar
William Stein committed
645
        func_type_args = []
646
        for i, arg_node in enumerate(self.args):
647
            name_declarator, type = arg_node.analyse(
648 649
                env, nonempty=nonempty,
                is_self_arg=(i == 0 and env.is_c_class_scope and 'staticmethod' not in env.directives))
William Stein's avatar
William Stein committed
650
            name = name_declarator.name
651 652 653 654 655
            if name in directive_locals:
                type_node = directive_locals[name]
                other_type = type_node.analyse_as_type(env)
                if other_type is None:
                    error(type_node.pos, "Not a type")
Robert Bradshaw's avatar
Robert Bradshaw committed
656
                elif (type is not PyrexTypes.py_object_type
657 658 659 660 661
                      and not type.same_as(other_type)):
                    error(self.base.pos, "Signature does not agree with previous declaration")
                    error(type_node.pos, "Previous declaration here")
                else:
                    type = other_type
William Stein's avatar
William Stein committed
662
            if name_declarator.cname:
663 664
                error(self.pos, "Function argument cannot have C name specification")
            if i == 0 and env.is_c_class_scope and type.is_unspecified:
665 666
                # fix the type of self
                type = env.parent_type
William Stein's avatar
William Stein committed
667 668 669 670 671
            # Turn *[] argument into **
            if type.is_array:
                type = PyrexTypes.c_ptr_type(type.base_type)
            # Catch attempted C-style func(void) decl
            if type.is_void:
Robert Bradshaw's avatar
Robert Bradshaw committed
672
                error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
William Stein's avatar
William Stein committed
673 674 675
            func_type_args.append(
                PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
            if arg_node.default:
676
                self.optional_arg_count += 1
677 678
            elif self.optional_arg_count:
                error(self.pos, "Non-default argument follows default argument")
679

William Stein's avatar
William Stein committed
680 681
        exc_val = None
        exc_check = 0
682
        if self.exception_check == '+':
683
            env.add_include_file('ios')         # for std::ios_base::failure
684
            env.add_include_file('new')         # for std::bad_alloc
685
            env.add_include_file('stdexcept')
686
            env.add_include_file('typeinfo')    # for std::bad_cast
687 688 689
        if (return_type.is_pyobject
                and (self.exception_value or self.exception_check)
                and self.exception_check != '+'):
690
            error(self.pos, "Exception clause not allowed for function returning Python object")
William Stein's avatar
William Stein committed
691 692
        else:
            if self.exception_value:
693
                self.exception_value = self.exception_value.analyse_const_expression(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
694 695
                if self.exception_check == '+':
                    exc_val_type = self.exception_value.type
696 697 698 699 700
                    if (not exc_val_type.is_error
                            and not exc_val_type.is_pyobject
                            and not (exc_val_type.is_cfunction
                                     and not exc_val_type.return_type.is_pyobject
                                     and not exc_val_type.args)):
Robert Bradshaw's avatar
Robert Bradshaw committed
701
                        error(self.exception_value.pos,
702
                              "Exception value must be a Python exception or cdef function with no arguments.")
703
                    exc_val = self.exception_value
Robert Bradshaw's avatar
Robert Bradshaw committed
704
                else:
705 706 707 708 709 710 711 712 713 714
                    self.exception_value = self.exception_value.coerce_to(
                        return_type, env).analyse_const_expression(env)
                    exc_val = self.exception_value.get_constant_c_result_code()
                    if exc_val is None:
                        raise InternalError(
                            "get_constant_c_result_code not implemented for %s" %
                            self.exception_value.__class__.__name__)
                    if not return_type.assignable_from(self.exception_value.type):
                        error(self.exception_value.pos,
                              "Exception value incompatible with function return type")
William Stein's avatar
William Stein committed
715
            exc_check = self.exception_check
716
        if return_type.is_cfunction:
717
            error(self.pos, "Function cannot return a function")
William Stein's avatar
William Stein committed
718
        func_type = PyrexTypes.CFuncType(
719
            return_type, func_type_args, self.has_varargs,
720 721 722 723 724 725
            optional_arg_count=self.optional_arg_count,
            exception_value=exc_val, exception_check=exc_check,
            calling_convention=self.base.calling_convention,
            nogil=self.nogil, with_gil=self.with_gil, is_overridable=self.overridable,
            is_const_method=self.is_const_method,
            templates=self.templates)
726

727
        if self.optional_arg_count:
728 729 730 731 732 733 734 735 736 737 738
            if func_type.is_fused:
                # This is a bit of a hack... When we need to create specialized CFuncTypes
                # on the fly because the cdef is defined in a pxd, we need to declare the specialized optional arg
                # struct
                def declare_opt_arg_struct(func_type, fused_cname):
                    self.declare_optional_arg_struct(func_type, env, fused_cname)

                func_type.declare_opt_arg_struct = declare_opt_arg_struct
            else:
                self.declare_optional_arg_struct(func_type, env)

739 740 741 742 743 744 745
        callspec = env.directives['callspec']
        if callspec:
            current = func_type.calling_convention
            if current and current != callspec:
                error(self.pos, "cannot have both '%s' and '%s' "
                      "calling conventions" % (current, callspec))
            func_type.calling_convention = callspec
William Stein's avatar
William Stein committed
746 747
        return self.base.analyse(func_type, env)

748 749 750 751 752 753 754 755 756 757 758
    def declare_optional_arg_struct(self, func_type, env, fused_cname=None):
        """
        Declares the optional argument struct (the struct used to hold the
        values for optional arguments). For fused cdef functions, this is
        deferred as analyse_declarations is called only once (on the fused
        cdef function).
        """
        scope = StructOrUnionScope()
        arg_count_member = '%sn' % Naming.pyrex_prefix
        scope.declare_var(arg_count_member, PyrexTypes.c_int_type, self.pos)

759 760
        for arg in func_type.args[len(func_type.args) - self.optional_arg_count:]:
            scope.declare_var(arg.name, arg.type, arg.pos, allow_pyobject=1)
761 762 763 764 765 766 767

        struct_cname = env.mangle(Naming.opt_arg_prefix, self.base.name)

        if fused_cname is not None:
            struct_cname = PyrexTypes.get_fused_cname(fused_cname, struct_cname)

        op_args_struct = env.global_scope().declare_struct_or_union(
768 769 770 771 772 773
            name=struct_cname,
            kind='struct',
            scope=scope,
            typedef_flag=0,
            pos=self.pos,
            cname=struct_cname)
774 775 776 777 778 779

        op_args_struct.defined_in_pxd = 1
        op_args_struct.used = 1

        func_type.op_arg_struct = PyrexTypes.c_ptr_type(op_args_struct.type)

William Stein's avatar
William Stein committed
780

781 782 783 784 785
class CConstDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

786
    def analyse(self, base_type, env, nonempty=0):
787 788 789 790
        if base_type.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        const = PyrexTypes.c_const_type(base_type)
791
        return self.base.analyse(const, env, nonempty=nonempty)
792 793


William Stein's avatar
William Stein committed
794 795 796 797 798 799
class CArgDeclNode(Node):
    # Item in a function declaration argument list.
    #
    # base_type      CBaseTypeNode
    # declarator     CDeclaratorNode
    # not_none       boolean            Tagged with 'not None'
800 801
    # or_none        boolean            Tagged with 'or None'
    # accept_none    boolean            Resolved boolean for not_none/or_none
William Stein's avatar
William Stein committed
802
    # default        ExprNode or None
803
    # default_value  PyObjectConst      constant for default value
804
    # annotation     ExprNode or None   Py3 function arg annotation
William Stein's avatar
William Stein committed
805
    # is_self_arg    boolean            Is the "self" arg of an extension type method
806
    # is_type_arg    boolean            Is the "class" arg of an extension type classmethod
807
    # is_kw_only     boolean            Is a keyword-only argument
808
    # is_dynamic     boolean            Non-literal arg stored inside CyFunction
809

810
    child_attrs = ["base_type", "declarator", "default", "annotation"]
811

William Stein's avatar
William Stein committed
812
    is_self_arg = 0
813
    is_type_arg = 0
814
    is_generic = 1
815 816 817
    kw_only = 0
    not_none = 0
    or_none = 0
818 819
    type = None
    name_declarator = None
820
    default_value = None
821
    annotation = None
822
    is_dynamic = 0
823

824
    def analyse(self, env, nonempty=0, is_self_arg=False):
825 826
        if is_self_arg:
            self.base_type.is_self_arg = self.is_self_arg = True
827
        if self.type is None:
Stefan Behnel's avatar
Stefan Behnel committed
828
            # The parser may misinterpret names as types. We fix that here.
829 830
            if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '':
                if nonempty:
831 832
                    if self.base_type.is_basic_c_type:
                        # char, short, long called "int"
833
                        type = self.base_type.analyse(env, could_be_name=True)
834
                        arg_name = type.empty_declaration_code()
835 836 837
                    else:
                        arg_name = self.base_type.name
                    self.declarator.name = EncodedString(arg_name)
838 839 840 841 842
                    self.base_type.name = None
                    self.base_type.is_basic_c_type = False
                could_be_name = True
            else:
                could_be_name = False
843
            self.base_type.is_arg = True
844
            base_type = self.base_type.analyse(env, could_be_name=could_be_name)
Robert Bradshaw's avatar
Robert Bradshaw committed
845
            if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name:
846
                self.declarator.name = self.base_type.arg_name
847

848 849
            # The parser is unable to resolve the ambiguity of [] as part of the
            # type (e.g. in buffers) or empty declarator (as with arrays).
850
            # This is only arises for empty multi-dimensional arrays.
851 852
            if (base_type.is_array
                    and isinstance(self.base_type, TemplatedTypeNode)
853 854 855 856 857 858
                    and isinstance(self.declarator, CArrayDeclaratorNode)):
                declarator = self.declarator
                while isinstance(declarator.base, CArrayDeclaratorNode):
                    declarator = declarator.base
                declarator.base = self.base_type.array_declarator
                base_type = base_type.base_type
859 860 861 862 863 864 865

            # inject type declaration from annotations
            if self.annotation and env.directives['annotation_typing'] and self.base_type.name is None:
                arg_type = self.inject_type_from_annotations(env)
                if arg_type is not None:
                    base_type = arg_type
            return self.declarator.analyse(base_type, env, nonempty=nonempty)
866
        else:
867
            return self.name_declarator, self.type
William Stein's avatar
William Stein committed
868

869 870 871
    def inject_type_from_annotations(self, env):
        annotation = self.annotation
        if not annotation:
872
            return None
873 874 875
        base_type, arg_type = _analyse_signature_annotation(annotation, env)
        if base_type is not None:
            self.base_type = base_type
876 877
        return arg_type

878 879 880 881 882 883 884 885 886
    def calculate_default_value_code(self, code):
        if self.default_value is None:
            if self.default:
                if self.default.is_literal:
                    # will not output any code, just assign the result_code
                    self.default.generate_evaluation_code(code)
                    return self.type.cast_code(self.default.result())
                self.default_value = code.get_argument_default_const(self.type)
        return self.default_value
887

888 889 890 891
    def annotate(self, code):
        if self.default:
            self.default.annotate(code)

892
    def generate_assignment_code(self, code, target=None, overloaded_assignment=False):
893 894 895 896 897 898 899
        default = self.default
        if default is None or default.is_literal:
            return
        if target is None:
            target = self.calculate_default_value_code(code)
        default.generate_evaluation_code(code)
        default.make_owned_reference(code)
900
        result = default.result() if overloaded_assignment else default.result_as(self.type)
901 902 903 904 905 906
        code.putln("%s = %s;" % (target, result))
        if self.type.is_pyobject:
            code.put_giveref(default.result())
        default.generate_post_assignment_code(code)
        default.free_temps(code)

William Stein's avatar
William Stein committed
907 908 909 910 911 912 913 914

class CBaseTypeNode(Node):
    # Abstract base class for C base type nodes.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #     Returns the type.
915

916 917
    def analyse_as_type(self, env):
        return self.analyse(env)
918

Stefan Behnel's avatar
Stefan Behnel committed
919

920 921
class CAnalysedBaseTypeNode(Node):
    # type            type
922

923
    child_attrs = []
924

925
    def analyse(self, env, could_be_name=False):
926
        return self.type
William Stein's avatar
William Stein committed
927

Stefan Behnel's avatar
Stefan Behnel committed
928

William Stein's avatar
William Stein committed
929 930 931 932 933 934
class CSimpleBaseTypeNode(CBaseTypeNode):
    # name             string
    # module_path      [string]     Qualifying name components
    # is_basic_c_type  boolean
    # signed           boolean
    # longness         integer
935
    # complex          boolean
William Stein's avatar
William Stein committed
936
    # is_self_arg      boolean      Is self argument of C method
937
    # ##is_type_arg      boolean      Is type argument of class method
William Stein's avatar
William Stein committed
938

939
    child_attrs = []
940
    arg_name = None   # in case the argument name was interpreted as a type
941 942 943
    module_path = []
    is_basic_c_type = False
    complex = False
944

945
    def analyse(self, env, could_be_name=False):
William Stein's avatar
William Stein committed
946
        # Return type descriptor.
947
        #print "CSimpleBaseTypeNode.analyse: is_self_arg =", self.is_self_arg ###
William Stein's avatar
William Stein committed
948 949 950 951 952 953 954 955 956
        type = None
        if self.is_basic_c_type:
            type = PyrexTypes.simple_c_type(self.signed, self.longness, self.name)
            if not type:
                error(self.pos, "Unrecognised type modifier combination")
        elif self.name == "object" and not self.module_path:
            type = py_object_type
        elif self.name is None:
            if self.is_self_arg and env.is_c_class_scope:
957
                #print "CSimpleBaseTypeNode.analyse: defaulting to parent type" ###
William Stein's avatar
William Stein committed
958
                type = env.parent_type
959 960
            ## elif self.is_type_arg and env.is_c_class_scope:
            ##     type = Builtin.type_type
William Stein's avatar
William Stein committed
961 962 963
            else:
                type = py_object_type
        else:
964
            if self.module_path:
965 966 967 968
                # Maybe it's a nested C++ class.
                scope = env
                for item in self.module_path:
                    entry = scope.lookup(item)
Robert Bradshaw's avatar
Robert Bradshaw committed
969
                    if entry is not None and entry.is_cpp_class:
970 971 972 973
                        scope = entry.type.scope
                    else:
                        scope = None
                        break
974

975 976 977 978 979
                if scope is None:
                    # Maybe it's a cimport.
                    scope = env.find_imported_module(self.module_path, self.pos)
                    if scope:
                        scope.fused_to_specific = env.fused_to_specific
980 981
            else:
                scope = env
982

William Stein's avatar
William Stein committed
983
            if scope:
984 985
                if scope.is_c_class_scope:
                    scope = scope.global_scope()
986 987 988 989

                type = scope.lookup_type(self.name)
                if type is not None:
                    pass
990 991 992
                elif could_be_name:
                    if self.is_self_arg and env.is_c_class_scope:
                        type = env.parent_type
993 994
                    ## elif self.is_type_arg and env.is_c_class_scope:
                    ##     type = Builtin.type_type
995 996
                    else:
                        type = py_object_type
997
                    self.arg_name = EncodedString(self.name)
William Stein's avatar
William Stein committed
998
                else:
Danilo Freitas's avatar
Danilo Freitas committed
999 1000 1001
                    if self.templates:
                        if not self.name in self.templates:
                            error(self.pos, "'%s' is not a type identifier" % self.name)
1002
                        type = PyrexTypes.TemplatePlaceholderType(self.name)
Danilo Freitas's avatar
Danilo Freitas committed
1003 1004
                    else:
                        error(self.pos, "'%s' is not a type identifier" % self.name)
1005
        if self.complex:
1006 1007 1008
            if not type.is_numeric or type.is_complex:
                error(self.pos, "can only complexify c numeric types")
            type = PyrexTypes.CComplexType(type)
1009
            type.create_declaration_utility_code(env)
1010 1011 1012 1013 1014 1015 1016 1017
        elif type is Builtin.complex_type:
            # Special case: optimise builtin complex type into C's
            # double complex.  The parser cannot do this (as for the
            # normal scalar types) as the user may have redeclared the
            # 'complex' type.  Testing for the exact type here works.
            type = PyrexTypes.c_double_complex_type
            type.create_declaration_utility_code(env)
            self.complex = True
William Stein's avatar
William Stein committed
1018 1019 1020 1021 1022
        if type:
            return type
        else:
            return PyrexTypes.error_type

1023
class MemoryViewSliceTypeNode(CBaseTypeNode):
1024

1025
    name = 'memoryview'
1026 1027
    child_attrs = ['base_type_node', 'axes']

1028
    def analyse(self, env, could_be_name=False):
1029 1030 1031 1032

        base_type = self.base_type_node.analyse(env)
        if base_type.is_error: return base_type

1033
        from . import MemoryView
1034 1035 1036

        try:
            axes_specs = MemoryView.get_axes_specs(env, self.axes)
1037
        except CompileError as e:
1038 1039 1040 1041
            error(e.position, e.message_only)
            self.type = PyrexTypes.ErrorType()
            return self.type

1042 1043 1044 1045
        if not MemoryView.validate_axes(self.pos, axes_specs):
            self.type = error_type
        else:
            self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
1046
            self.type.validate_memslice_dtype(self.pos)
1047
            self.use_memview_utilities(env)
1048

1049
        return self.type
1050

1051
    def use_memview_utilities(self, env):
1052
        from . import MemoryView
1053 1054 1055
        env.use_utility_code(MemoryView.view_utility_code)


Robert Bradshaw's avatar
Robert Bradshaw committed
1056
class CNestedBaseTypeNode(CBaseTypeNode):
1057
    # For C++ classes that live inside other C++ classes.
Robert Bradshaw's avatar
Robert Bradshaw committed
1058 1059 1060

    # name             string
    # base_type        CBaseTypeNode
Stefan Behnel's avatar
Stefan Behnel committed
1061

Robert Bradshaw's avatar
Robert Bradshaw committed
1062
    child_attrs = ['base_type']
Stefan Behnel's avatar
Stefan Behnel committed
1063

1064
    def analyse(self, env, could_be_name=None):
Robert Bradshaw's avatar
Robert Bradshaw committed
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
        base_type = self.base_type.analyse(env)
        if base_type is PyrexTypes.error_type:
            return PyrexTypes.error_type
        if not base_type.is_cpp_class:
            error(self.pos, "'%s' is not a valid type scope" % base_type)
            return PyrexTypes.error_type
        type_entry = base_type.scope.lookup_here(self.name)
        if not type_entry or not type_entry.is_type:
            error(self.pos, "'%s.%s' is not a type identifier" % (base_type, self.name))
            return PyrexTypes.error_type
        return type_entry.type

1077

Danilo Freitas's avatar
Danilo Freitas committed
1078
class TemplatedTypeNode(CBaseTypeNode):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1079
    #  After parsing:
1080 1081
    #  positional_args  [ExprNode]        List of positional arguments
    #  keyword_args     DictNode          Keyword arguments
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1082 1083 1084
    #  base_type_node   CBaseTypeNode

    #  After analysis:
1085
    #  type             PyrexTypes.BufferType or PyrexTypes.CppClassType  ...containing the right options
1086 1087 1088

    child_attrs = ["base_type_node", "positional_args",
                   "keyword_args", "dtype_node"]
1089

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1090
    dtype_node = None
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1091 1092

    name = None
1093

1094
    def analyse(self, env, could_be_name=False, base_type=None):
1095 1096
        if base_type is None:
            base_type = self.base_type_node.analyse(env)
1097
        if base_type.is_error: return base_type
1098

1099
        if base_type.is_cpp_class and base_type.is_template_type():
1100
            # Templated class
1101
            if self.keyword_args and self.keyword_args.key_value_pairs:
Stefan Behnel's avatar
Stefan Behnel committed
1102
                error(self.pos, "c++ templates cannot take keyword arguments")
1103 1104 1105 1106
                self.type = PyrexTypes.error_type
            else:
                template_types = []
                for template_node in self.positional_args:
1107 1108 1109 1110 1111
                    type = template_node.analyse_as_type(env)
                    if type is None:
                        error(template_node.pos, "unknown type in template argument")
                        return error_type
                    template_types.append(type)
1112
                self.type = base_type.specialize_here(self.pos, template_types)
1113

1114 1115
        elif base_type.is_pyobject:
            # Buffer
1116
            from . import Buffer
1117

1118 1119 1120 1121 1122 1123
            options = Buffer.analyse_buffer_options(
                self.pos,
                env,
                self.positional_args,
                self.keyword_args,
                base_type.buffer_defaults)
1124

Robert Bradshaw's avatar
Robert Bradshaw committed
1125 1126
            if sys.version_info[0] < 3:
                # Py 2.x enforces byte strings as keyword arguments ...
1127 1128
                options = dict([(name.encode('ASCII'), value)
                                for name, value in options.items()])
Stefan Behnel's avatar
Stefan Behnel committed
1129

1130
            self.type = PyrexTypes.BufferType(base_type, **options)
1131 1132
            if has_np_pythran(env) and is_pythran_buffer(self.type):
                self.type = PyrexTypes.PythranExpr(pythran_type(self.type), self.type)
1133

1134 1135
        else:
            # Array
1136
            empty_declarator = CNameDeclaratorNode(self.pos, name="", cname=None)
1137 1138 1139 1140
            if len(self.positional_args) > 1 or self.keyword_args.key_value_pairs:
                error(self.pos, "invalid array declaration")
                self.type = PyrexTypes.error_type
            else:
1141
                # It would be nice to merge this class with CArrayDeclaratorNode,
1142 1143 1144 1145 1146
                # but arrays are part of the declaration, not the type...
                if not self.positional_args:
                    dimension = None
                else:
                    dimension = self.positional_args[0]
1147 1148 1149 1150
                self.array_declarator = CArrayDeclaratorNode(
                    self.pos,
                    base=empty_declarator,
                    dimension=dimension)
1151
                self.type = self.array_declarator.analyse(base_type, env)[1]
1152

1153 1154 1155
        if self.type.is_fused and env.fused_to_specific:
            self.type = self.type.specialize(env.fused_to_specific)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1156
        return self.type
William Stein's avatar
William Stein committed
1157

1158

William Stein's avatar
William Stein committed
1159 1160 1161
class CComplexBaseTypeNode(CBaseTypeNode):
    # base_type   CBaseTypeNode
    # declarator  CDeclaratorNode
1162

1163 1164
    child_attrs = ["base_type", "declarator"]

1165
    def analyse(self, env, could_be_name=False):
1166
        base = self.base_type.analyse(env, could_be_name)
William Stein's avatar
William Stein committed
1167 1168 1169 1170
        _, type = self.declarator.analyse(base, env)
        return type


1171 1172 1173 1174 1175
class CTupleBaseTypeNode(CBaseTypeNode):
    # components [CBaseTypeNode]

    child_attrs = ["components"]

Robert Bradshaw's avatar
Robert Bradshaw committed
1176
    def analyse(self, env, could_be_name=False):
1177 1178 1179 1180
        component_types = []
        for c in self.components:
            type = c.analyse(env)
            if type.is_pyobject:
1181 1182
                error(c.pos, "Tuple types can't (yet) contain Python objects.")
                return error_type
1183
            component_types.append(type)
1184
        entry = env.declare_tuple_type(self.pos, component_types)
Robert Bradshaw's avatar
Robert Bradshaw committed
1185
        entry.used = True
1186
        return entry.type
1187 1188


1189 1190 1191 1192 1193 1194
class FusedTypeNode(CBaseTypeNode):
    """
    Represents a fused type in a ctypedef statement:

        ctypedef cython.fused_type(int, long, long long) integral

Mark Florisson's avatar
Mark Florisson committed
1195
    name            str                     name of this fused type
1196 1197 1198 1199 1200
    types           [CSimpleBaseTypeNode]   is the list of types to be fused
    """

    child_attrs = []

Mark Florisson's avatar
Mark Florisson committed
1201 1202 1203 1204 1205 1206 1207
    def analyse_declarations(self, env):
        type = self.analyse(env)
        entry = env.declare_typedef(self.name, type, self.pos)

        # Omit the typedef declaration that self.declarator would produce
        entry.in_cinclude = True

1208
    def analyse(self, env, could_be_name=False):
Mark Florisson's avatar
Mark Florisson committed
1209 1210 1211
        types = []
        for type_node in self.types:
            type = type_node.analyse_as_type(env)
1212

Mark Florisson's avatar
Mark Florisson committed
1213 1214 1215
            if not type:
                error(type_node.pos, "Not a type")
                continue
1216

Mark Florisson's avatar
Mark Florisson committed
1217
            if type in types:
1218 1219
                error(type_node.pos, "Type specified multiple times")
            else:
Mark Florisson's avatar
Mark Florisson committed
1220
                types.append(type)
1221

1222 1223
        # if len(self.types) == 1:
        #     return types[0]
Mark Florisson's avatar
Mark Florisson committed
1224 1225

        return PyrexTypes.FusedType(types, name=self.name)
1226

1227

Robert Bradshaw's avatar
Robert Bradshaw committed
1228 1229 1230 1231 1232
class CConstTypeNode(CBaseTypeNode):
    # base_type     CBaseTypeNode

    child_attrs = ["base_type"]

1233
    def analyse(self, env, could_be_name=False):
Robert Bradshaw's avatar
Robert Bradshaw committed
1234 1235 1236 1237 1238 1239 1240
        base = self.base_type.analyse(env, could_be_name)
        if base.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        return PyrexTypes.c_const_type(base)


William Stein's avatar
William Stein committed
1241 1242 1243 1244 1245 1246
class CVarDefNode(StatNode):
    #  C variable definition or forward/extern function declaration.
    #
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarators   [CDeclaratorNode]
1247
    #  in_pxd        boolean
Stefan Behnel's avatar
Stefan Behnel committed
1248
    #  api           boolean
1249
    #  overridable   boolean        whether it is a cpdef
1250
    #  modifiers     ['inline']
1251

1252
    #  decorators    [cython.locals(...)] or None
1253
    #  directive_locals { string : NameNode } locals defined by cython.locals(...)
1254 1255

    child_attrs = ["base_type", "declarators"]
1256

Robert Bradshaw's avatar
Robert Bradshaw committed
1257
    decorators = None
1258
    directive_locals = None
1259

1260
    def analyse_declarations(self, env, dest_scope=None):
1261 1262
        if self.directive_locals is None:
            self.directive_locals = {}
William Stein's avatar
William Stein committed
1263 1264
        if not dest_scope:
            dest_scope = env
1265
        self.dest_scope = dest_scope
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280

        if self.declarators:
            templates = self.declarators[0].analyse_templates()
        else:
            templates = None
        if templates is not None:
            if self.visibility != 'extern':
                error(self.pos, "Only extern functions allowed")
            if len(self.declarators) > 1:
                error(self.declarators[1].pos, "Can't multiply declare template types")
            env = TemplateScope('func_template', env)
            env.directives = env.outer_scope.directives
            for template_param in templates:
                env.declare_type(template_param.name, template_param, self.pos)

William Stein's avatar
William Stein committed
1281
        base_type = self.base_type.analyse(env)
1282

1283 1284
        if base_type.is_fused and not self.in_pxd and (env.is_c_class_scope or
                                                       env.is_module_scope):
1285 1286 1287
            error(self.pos, "Fused types not allowed here")
            return error_type

1288
        self.entry = None
1289
        visibility = self.visibility
1290

William Stein's avatar
William Stein committed
1291
        for declarator in self.declarators:
1292

1293
            if (len(self.declarators) > 1
1294 1295 1296 1297 1298
                    and not isinstance(declarator, CNameDeclaratorNode)
                    and env.directives['warn.multiple_declarators']):
                warning(
                    declarator.pos,
                    "Non-trivial type declarators in shared declaration (e.g. mix of pointers and values). "
1299
                    "Each pointer declaration should be on its own line.", 1)
1300

1301
            create_extern_wrapper = (self.overridable
1302 1303
                                     and self.visibility == 'extern'
                                     and env.is_module_scope)
1304 1305
            if create_extern_wrapper:
                declarator.overridable = False
1306 1307 1308 1309
            if isinstance(declarator, CFuncDeclaratorNode):
                name_declarator, type = declarator.analyse(base_type, env, directive_locals=self.directive_locals)
            else:
                name_declarator, type = declarator.analyse(base_type, env)
William Stein's avatar
William Stein committed
1310
            if not type.is_complete():
1311
                if not (self.visibility == 'extern' and type.is_array or type.is_memoryviewslice):
1312
                    error(declarator.pos, "Variable type '%s' is incomplete" % type)
William Stein's avatar
William Stein committed
1313
            if self.visibility == 'extern' and type.is_pyobject:
1314
                error(declarator.pos, "Python object cannot be declared extern")
William Stein's avatar
William Stein committed
1315 1316
            name = name_declarator.name
            cname = name_declarator.cname
1317 1318 1319
            if name == '':
                error(declarator.pos, "Missing name in declaration.")
                return
1320 1321
            if type.is_reference and self.visibility != 'extern':
                error(declarator.pos, "C++ references cannot be declared; use a pointer instead")
William Stein's avatar
William Stein committed
1322
            if type.is_cfunction:
1323 1324
                if 'staticmethod' in env.directives:
                    type.is_static_method = True
1325 1326
                self.entry = dest_scope.declare_cfunction(
                    name, type, declarator.pos,
1327 1328
                    cname=cname, visibility=self.visibility, in_pxd=self.in_pxd,
                    api=self.api, modifiers=self.modifiers, overridable=self.overridable)
1329 1330
                if self.entry is not None:
                    self.entry.directive_locals = copy.copy(self.directive_locals)
1331
                if create_extern_wrapper:
1332
                    self.entry.type.create_to_py_utility_code(env)
1333
                    self.entry.create_wrapper = True
William Stein's avatar
William Stein committed
1334
            else:
1335
                if self.directive_locals:
1336
                    error(self.pos, "Decorators can only be followed by functions")
1337 1338 1339 1340
                self.entry = dest_scope.declare_var(
                    name, type, declarator.pos,
                    cname=cname, visibility=visibility, in_pxd=self.in_pxd,
                    api=self.api, is_cdef=1)
1341 1342
                if Options.docstrings:
                    self.entry.doc = embed_position(self.pos, self.doc)
1343

William Stein's avatar
William Stein committed
1344 1345 1346 1347 1348 1349

class CStructOrUnionDefNode(StatNode):
    #  name          string
    #  cname         string or None
    #  kind          "struct" or "union"
    #  typedef_flag  boolean
1350
    #  visibility    "public" or "private"
1351
    #  api           boolean
Stefan Behnel's avatar
Stefan Behnel committed
1352
    #  in_pxd        boolean
William Stein's avatar
William Stein committed
1353 1354
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1355
    #  packed        boolean
1356

1357
    child_attrs = ["attributes"]
Vitja Makarov's avatar
Vitja Makarov committed
1358

1359
    def declare(self, env, scope=None):
William Stein's avatar
William Stein committed
1360 1361
        self.entry = env.declare_struct_or_union(
            self.name, self.kind, scope, self.typedef_flag, self.pos,
1362 1363
            self.cname, visibility=self.visibility, api=self.api,
            packed=self.packed)
1364 1365 1366 1367 1368 1369

    def analyse_declarations(self, env):
        scope = None
        if self.attributes is not None:
            scope = StructOrUnionScope(self.name)
        self.declare(env, scope)
William Stein's avatar
William Stein committed
1370
        if self.attributes is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1371 1372
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
William Stein's avatar
William Stein committed
1373 1374
            for attr in self.attributes:
                attr.analyse_declarations(env, scope)
1375 1376 1377 1378
            if self.visibility != 'extern':
                for attr in scope.var_entries:
                    type = attr.type
                    while type.is_array:
1379 1380
                        type = type.base_type
                    if type == self.entry.type:
1381
                        error(attr.pos, "Struct cannot contain itself as a member.")
1382

William Stein's avatar
William Stein committed
1383
    def analyse_expressions(self, env):
1384
        return self
1385

William Stein's avatar
William Stein committed
1386 1387 1388 1389
    def generate_execution_code(self, code):
        pass


1390
class CppClassNode(CStructOrUnionDefNode, BlockNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
1391 1392 1393

    #  name          string
    #  cname         string or None
1394
    #  visibility    "extern"
Robert Bradshaw's avatar
Robert Bradshaw committed
1395 1396 1397
    #  in_pxd        boolean
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1398
    #  base_classes  [CBaseTypeNode]
1399
    #  templates     [(string, bool)] or None
1400 1401 1402
    #  decorators    [DecoratorNode] or None

    decorators = None
Robert Bradshaw's avatar
Robert Bradshaw committed
1403

1404 1405 1406 1407
    def declare(self, env):
        if self.templates is None:
            template_types = None
        else:
1408 1409 1410 1411 1412
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
                              for template_name, required in self.templates]
            num_optional_templates = sum(not required for _, required in self.templates)
            if num_optional_templates and not all(required for _, required in self.templates[:-num_optional_templates]):
                error(self.pos, "Required template parameters must precede optional template parameters.")
1413
        self.entry = env.declare_cpp_class(
1414 1415
            self.name, None, self.pos, self.cname,
            base_classes=[], visibility=self.visibility, templates=template_types)
1416

Robert Bradshaw's avatar
Robert Bradshaw committed
1417
    def analyse_declarations(self, env):
1418 1419 1420 1421 1422 1423
        if self.templates is None:
            template_types = template_names = None
        else:
            template_names = [template_name for template_name, _ in self.templates]
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
                              for template_name, required in self.templates]
Robert Bradshaw's avatar
Robert Bradshaw committed
1424
        scope = None
1425
        if self.attributes is not None:
1426
            scope = CppClassScope(self.name, env, templates=template_names)
1427 1428 1429 1430 1431 1432
        def base_ok(base_class):
            if base_class.is_cpp_class or base_class.is_struct:
                return True
            else:
                error(self.pos, "Base class '%s' not a struct or class." % base_class)
        base_class_types = filter(base_ok, [b.analyse(scope or env) for b in self.base_classes])
1433
        self.entry = env.declare_cpp_class(
1434
            self.name, scope, self.pos,
1435
            self.cname, base_class_types, visibility=self.visibility, templates=template_types)
1436 1437
        if self.entry is None:
            return
Danilo Freitas's avatar
Danilo Freitas committed
1438
        self.entry.is_cpp_class = 1
Stefan Behnel's avatar
Stefan Behnel committed
1439 1440
        if scope is not None:
            scope.type = self.entry.type
1441
        defined_funcs = []
1442 1443 1444 1445 1446 1447 1448
        def func_attributes(attributes):
            for attr in attributes:
                if isinstance(attr, CFuncDefNode):
                    yield attr
                elif isinstance(attr, CompilerDirectivesNode):
                    for sub_attr in func_attributes(attr.body.stats):
                        yield sub_attr
Robert Bradshaw's avatar
Robert Bradshaw committed
1449 1450 1451 1452
        if self.attributes is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for attr in self.attributes:
1453 1454 1455
                declare = getattr(attr, 'declare', None)
                if declare:
                    attr.declare(scope)
Robert Bradshaw's avatar
Robert Bradshaw committed
1456
                attr.analyse_declarations(scope)
1457 1458 1459
            for func in func_attributes(self.attributes):
                defined_funcs.append(func)
                if self.templates is not None:
1460
                    func.template_declaration = "template <typename %s>" % ", typename ".join(template_names)
1461 1462 1463 1464
        self.body = StatListNode(self.pos, stats=defined_funcs)
        self.scope = scope

    def analyse_expressions(self, env):
1465 1466
        self.body = self.body.analyse_expressions(self.entry.type.scope)
        return self
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476

    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(self.entry.type.scope, code)

    def generate_execution_code(self, code):
        self.body.generate_execution_code(code)

    def annotate(self, code):
        self.body.annotate(code)

Robert Bradshaw's avatar
Robert Bradshaw committed
1477

William Stein's avatar
William Stein committed
1478 1479 1480 1481 1482
class CEnumDefNode(StatNode):
    #  name           string or None
    #  cname          string or None
    #  items          [CEnumDefItemNode]
    #  typedef_flag   boolean
1483
    #  visibility     "public" or "private" or "extern"
1484
    #  api            boolean
Stefan Behnel's avatar
Stefan Behnel committed
1485
    #  in_pxd         boolean
1486
    #  create_wrapper boolean
William Stein's avatar
William Stein committed
1487
    #  entry          Entry
Robert Bradshaw's avatar
Robert Bradshaw committed
1488

1489
    child_attrs = ["items"]
1490

1491
    def declare(self, env):
1492 1493 1494 1495
         self.entry = env.declare_enum(
             self.name, self.pos,
             cname=self.cname, typedef_flag=self.typedef_flag,
             visibility=self.visibility, api=self.api,
1496
             create_wrapper=self.create_wrapper)
1497

William Stein's avatar
William Stein committed
1498
    def analyse_declarations(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
1499 1500 1501 1502 1503
        if self.items is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for item in self.items:
                item.analyse_declarations(env, self.entry)
William Stein's avatar
William Stein committed
1504

1505
    def analyse_expressions(self, env):
1506
        return self
1507

William Stein's avatar
William Stein committed
1508
    def generate_execution_code(self, code):
1509
        if self.visibility == 'public' or self.api:
1510
            code.mark_pos(self.pos)
1511
            temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
1512 1513
            for item in self.entry.enum_values:
                code.putln("%s = PyInt_FromLong(%s); %s" % (
1514 1515 1516
                    temp,
                    item.cname,
                    code.error_goto_if_null(temp, item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1517
                code.put_gotref(temp)
1518
                code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % (
1519 1520 1521 1522
                    Naming.moddict_cname,
                    item.name,
                    temp,
                    code.error_goto(item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1523
                code.put_decref_clear(temp, PyrexTypes.py_object_type)
1524
            code.funcstate.release_temp(temp)
William Stein's avatar
William Stein committed
1525 1526 1527 1528 1529 1530


class CEnumDefItemNode(StatNode):
    #  name     string
    #  cname    string or None
    #  value    ExprNode or None
1531

1532 1533
    child_attrs = ["value"]

William Stein's avatar
William Stein committed
1534 1535
    def analyse_declarations(self, env, enum_entry):
        if self.value:
1536
            self.value = self.value.analyse_const_expression(env)
1537 1538
            if not self.value.type.is_int:
                self.value = self.value.coerce_to(PyrexTypes.c_int_type, env)
1539
                self.value = self.value.analyse_const_expression(env)
1540 1541 1542 1543 1544
        entry = env.declare_const(
            self.name, enum_entry.type,
            self.value, self.pos, cname=self.cname,
            visibility=enum_entry.visibility, api=enum_entry.api,
            create_wrapper=enum_entry.create_wrapper and enum_entry.name is None)
William Stein's avatar
William Stein committed
1545
        enum_entry.enum_values.append(entry)
1546
        if enum_entry.name:
1547
            enum_entry.type.values.append(entry.name)
William Stein's avatar
William Stein committed
1548 1549 1550


class CTypeDefNode(StatNode):
Stefan Behnel's avatar
Stefan Behnel committed
1551 1552 1553
    #  base_type    CBaseTypeNode
    #  declarator   CDeclaratorNode
    #  visibility   "public" or "private"
1554
    #  api          boolean
Stefan Behnel's avatar
Stefan Behnel committed
1555
    #  in_pxd       boolean
1556 1557

    child_attrs = ["base_type", "declarator"]
1558

William Stein's avatar
William Stein committed
1559 1560 1561 1562 1563
    def analyse_declarations(self, env):
        base = self.base_type.analyse(env)
        name_declarator, type = self.declarator.analyse(base, env)
        name = name_declarator.name
        cname = name_declarator.cname
1564

1565 1566 1567
        entry = env.declare_typedef(
            name, type, self.pos,
            cname=cname, visibility=self.visibility, api=self.api)
1568

Mark Florisson's avatar
Mark Florisson committed
1569
        if type.is_fused:
1570 1571
            entry.in_cinclude = True

Mark Florisson's avatar
Mark Florisson committed
1572 1573
        if self.in_pxd and not env.in_cinclude:
            entry.defined_in_pxd = 1
1574

William Stein's avatar
William Stein committed
1575
    def analyse_expressions(self, env):
1576 1577
        return self

William Stein's avatar
William Stein committed
1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
    def generate_execution_code(self, code):
        pass


class FuncDefNode(StatNode, BlockNode):
    #  Base class for function definition nodes.
    #
    #  return_type     PyrexType
    #  #filename        string        C name of filename string const
    #  entry           Symtab.Entry
1588
    #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
Vitja Makarov's avatar
Vitja Makarov committed
1589
    #  needs_outer_scope boolean      Whether or not this function requires outer scope
1590
    #  pymethdef_required boolean     Force Python method struct generation
1591 1592
    #  directive_locals { string : ExprNode } locals defined by cython.locals(...)
    #  directive_returns [ExprNode] type defined by cython.returns(...)
1593 1594 1595 1596 1597
    #  star_arg      PyArgDeclNode or None  * argument
    #  starstar_arg  PyArgDeclNode or None  ** argument
    #
    #  is_async_def  boolean          is a Coroutine function
    #
1598 1599 1600 1601 1602
    #  has_fused_arguments  boolean
    #       Whether this cdef function has fused parameters. This is needed
    #       by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
    #       with fused argument types with a FusedCFuncDefNode

1603
    py_func = None
1604
    needs_closure = False
Vitja Makarov's avatar
Vitja Makarov committed
1605
    needs_outer_scope = False
1606
    pymethdef_required = False
1607
    is_generator = False
1608
    is_generator_body = False
1609
    is_async_def = False
Robert Bradshaw's avatar
Robert Bradshaw committed
1610
    modifiers = []
1611
    has_fused_arguments = False
1612 1613
    star_arg = None
    starstar_arg = None
1614
    is_cyfunction = False
1615
    code_object = None
1616

1617
    def analyse_default_values(self, env):
1618
        default_seen = 0
1619 1620
        for arg in self.args:
            if arg.default:
1621
                default_seen = 1
1622
                if arg.is_generic:
1623
                    arg.default = arg.default.analyse_types(env)
1624
                    arg.default = arg.default.coerce_to(arg.type, env)
1625
                else:
1626
                    error(arg.pos, "This argument cannot have a default value")
1627
                    arg.default = None
1628 1629 1630 1631
            elif arg.kw_only:
                default_seen = 1
            elif default_seen:
                error(arg.pos, "Non-default argument following default argument")
1632

1633 1634 1635 1636 1637
    def analyse_annotations(self, env):
        for arg in self.args:
            if arg.annotation:
                arg.annotation = arg.annotation.analyse_types(env)

1638
    def align_argument_type(self, env, arg):
1639
        # @cython.locals()
1640
        directive_locals = self.directive_locals
1641
        orig_type = arg.type
1642 1643 1644
        if arg.name in directive_locals:
            type_node = directive_locals[arg.name]
            other_type = type_node.analyse_as_type(env)
1645
        elif isinstance(arg, CArgDeclNode) and arg.annotation and env.directives['annotation_typing']:
1646 1647
            type_node = arg.annotation
            other_type = arg.inject_type_from_annotations(env)
1648 1649
            if other_type is None:
                return arg
1650 1651 1652 1653
        else:
            return arg
        if other_type is None:
            error(type_node.pos, "Not a type")
1654
        elif orig_type is not py_object_type and not orig_type.same_as(other_type):
1655 1656 1657 1658
            error(arg.base_type.pos, "Signature does not agree with previous declaration")
            error(type_node.pos, "Previous declaration here")
        else:
            arg.type = other_type
1659 1660
        return arg

1661 1662
    def need_gil_acquisition(self, lenv):
        return 0
1663

1664 1665
    def create_local_scope(self, env):
        genv = env
1666
        while genv.is_py_class_scope or genv.is_c_class_scope:
1667
            genv = genv.outer_scope
1668
        if self.needs_closure:
1669
            lenv = ClosureScope(name=self.entry.name,
1670 1671
                                outer_scope=genv,
                                parent_scope=env,
1672
                                scope_name=self.entry.cname)
1673
        else:
1674 1675 1676
            lenv = LocalScope(name=self.entry.name,
                              outer_scope=genv,
                              parent_scope=env)
William Stein's avatar
William Stein committed
1677
        lenv.return_type = self.return_type
1678 1679 1680
        type = self.entry.type
        if type.is_cfunction:
            lenv.nogil = type.nogil and not type.with_gil
1681
        self.local_scope = lenv
1682
        lenv.directives = env.directives
1683
        return lenv
1684

1685 1686 1687
    def generate_function_body(self, env, code):
        self.body.generate_execution_code(code)

1688
    def generate_function_definitions(self, env, code):
1689
        from . import Buffer
1690
        if self.return_type.is_memoryviewslice:
1691
            from . import MemoryView
1692 1693

        lenv = self.local_scope
Vitja Makarov's avatar
Vitja Makarov committed
1694
        if lenv.is_closure_scope and not lenv.is_passthrough:
1695 1696 1697 1698 1699
            outer_scope_cname = "%s->%s" % (Naming.cur_scope_cname,
                                            Naming.outer_scope_cname)
        else:
            outer_scope_cname = Naming.outer_scope_cname
        lenv.mangle_closure_cnames(outer_scope_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
1700 1701
        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)
Stefan Behnel's avatar
Stefan Behnel committed
1702
        # generate lambda function definitions
1703
        self.generate_lambda_definitions(lenv, code)
1704

1705 1706
        is_getbuffer_slot = (self.entry.name == "__getbuffer__" and
                             self.entry.scope.is_c_class_scope)
1707 1708 1709
        is_releasebuffer_slot = (self.entry.name == "__releasebuffer__" and
                                 self.entry.scope.is_c_class_scope)
        is_buffer_slot = is_getbuffer_slot or is_releasebuffer_slot
1710
        if is_buffer_slot:
1711 1712
            if 'cython_unused' not in self.modifiers:
                self.modifiers = self.modifiers + ['cython_unused']
1713

1714
        preprocessor_guard = self.get_preprocessor_guard()
1715

1716
        profile = code.globalstate.directives['profile']
1717 1718
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
1719 1720
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("Profile", "Profile.c"))
1721

1722
        # Generate C code for header and body of function
1723
        code.enter_cfunc_scope(lenv)
1724
        code.return_from_error_cleanup_label = code.new_label()
1725
        code.funcstate.gil_owned = not lenv.nogil
1726

William Stein's avatar
William Stein committed
1727
        # ----- Top-level constants used by this function
1728
        code.mark_pos(self.pos)
1729
        self.generate_cached_builtins_decls(lenv, code)
William Stein's avatar
William Stein committed
1730 1731
        # ----- Function header
        code.putln("")
1732 1733 1734 1735

        if preprocessor_guard:
            code.putln(preprocessor_guard)

1736 1737
        with_pymethdef = (self.needs_assignment_synthesis(env, code) or
                          self.pymethdef_required)
1738
        if self.py_func:
1739 1740 1741
            self.py_func.generate_function_header(
                code, with_pymethdef=with_pymethdef, proto_only=True)
        self.generate_function_header(code, with_pymethdef=with_pymethdef)
William Stein's avatar
William Stein committed
1742
        # ----- Local variable declarations
1743 1744 1745 1746
        # Find function scope
        cenv = env
        while cenv.is_py_class_scope or cenv.is_c_class_scope:
            cenv = cenv.outer_scope
Vitja Makarov's avatar
Vitja Makarov committed
1747
        if self.needs_closure:
1748
            code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
Robert Bradshaw's avatar
Robert Bradshaw committed
1749
            code.putln(";")
Vitja Makarov's avatar
Vitja Makarov committed
1750 1751 1752 1753
        elif self.needs_outer_scope:
            if lenv.is_passthrough:
                code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
                code.putln(";")
1754
            code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
1755
            code.putln(";")
William Stein's avatar
William Stein committed
1756
        self.generate_argument_declarations(lenv, code)
1757

1758
        for entry in lenv.var_entries:
1759
            if not (entry.in_closure or entry.is_arg):
1760
                code.put_var_declaration(entry)
1761

1762
        # Initialize the return variable __pyx_r
William Stein's avatar
William Stein committed
1763 1764
        init = ""
        if not self.return_type.is_void:
1765 1766
            if self.return_type.is_pyobject:
                init = " = NULL"
1767
            elif self.return_type.is_memoryviewslice:
1768
                init = ' = ' + MemoryView.memslice_entry_init
1769

1770 1771 1772
            code.putln("%s%s;" % (
                self.return_type.declaration_code(Naming.retval_cname),
                init))
1773

1774
        tempvardecl_code = code.insertion_point()
William Stein's avatar
William Stein committed
1775
        self.generate_keyword_list(code)
1776

William Stein's avatar
William Stein committed
1777 1778
        # ----- Extern library function declarations
        lenv.generate_library_function_declarations(code)
1779

1780
        # ----- GIL acquisition
1781
        acquire_gil = self.acquire_gil
1782

Mark Florisson's avatar
Mark Florisson committed
1783 1784
        # See if we need to acquire the GIL for variable declarations, or for
        # refnanny only
1785

1786 1787 1788
        # Closures are not currently possible for cdef nogil functions,
        # but check them anyway
        have_object_args = self.needs_closure or self.needs_outer_scope
Mark Florisson's avatar
Mark Florisson committed
1789 1790 1791 1792 1793
        for arg in lenv.arg_entries:
            if arg.type.is_pyobject:
                have_object_args = True
                break

1794 1795
        used_buffer_entries = [entry for entry in lenv.buffer_entries if entry.used]

Mark Florisson's avatar
Mark Florisson committed
1796
        acquire_gil_for_var_decls_only = (
1797 1798
            lenv.nogil and lenv.has_with_gil_block and
            (have_object_args or used_buffer_entries))
Mark Florisson's avatar
Mark Florisson committed
1799 1800

        acquire_gil_for_refnanny_only = (
1801 1802
            lenv.nogil and lenv.has_with_gil_block and not
            acquire_gil_for_var_decls_only)
Mark Florisson's avatar
Mark Florisson committed
1803 1804

        use_refnanny = not lenv.nogil or lenv.has_with_gil_block
1805 1806 1807

        if acquire_gil or acquire_gil_for_var_decls_only:
            code.put_ensure_gil()
1808
            code.funcstate.gil_owned = True
1809 1810
        elif lenv.nogil and lenv.has_with_gil_block:
            code.declare_gilstate()
1811

1812
        if profile or linetrace:
1813 1814 1815 1816 1817
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                tempvardecl_code.put_trace_declarations()
                code_object = self.code_object.calculate_result_code(code) if self.code_object else None
                code.put_trace_frame_init(code_object)
1818

1819
        # ----- set up refnanny
1820
        if use_refnanny:
1821
            tempvardecl_code.put_declare_refcount_context()
1822 1823
            code.put_setup_refcount_context(
                self.entry.name, acquire_gil=acquire_gil_for_refnanny_only)
Mark Florisson's avatar
Mark Florisson committed
1824

1825
        # ----- Automatic lead-ins for certain special functions
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1826 1827
        if is_getbuffer_slot:
            self.getbuffer_init(code)
1828
        # ----- Create closure scope object
Robert Bradshaw's avatar
Robert Bradshaw committed
1829
        if self.needs_closure:
1830
            tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__')
1831 1832 1833 1834
            slot_func_cname = TypeSlots.get_slot_function(lenv.scope_class.type.scope, tp_slot)
            if not slot_func_cname:
                slot_func_cname = '%s->tp_new' % lenv.scope_class.type.typeptr_cname
            code.putln("%s = (%s)%s(%s, %s, NULL);" % (
1835
                Naming.cur_scope_cname,
1836
                lenv.scope_class.type.empty_declaration_code(),
1837
                slot_func_cname,
1838
                lenv.scope_class.type.typeptr_cname,
1839 1840
                Naming.empty_tuple))
            code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
1841 1842 1843 1844 1845 1846 1847
            # Scope unconditionally DECREFed on return.
            code.putln("%s = %s;" % (
                Naming.cur_scope_cname,
                lenv.scope_class.type.cast_code("Py_None")));
            code.put_incref("Py_None", py_object_type);
            code.putln(code.error_goto(self.pos))
            code.putln("} else {")
Robert Bradshaw's avatar
Robert Bradshaw committed
1848
            code.put_gotref(Naming.cur_scope_cname)
1849
            code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
1850
            # Note that it is unsafe to decref the scope at this point.
Vitja Makarov's avatar
Vitja Makarov committed
1851
        if self.needs_outer_scope:
1852 1853 1854
            if self.is_cyfunction:
                code.putln("%s = (%s) __Pyx_CyFunction_GetClosure(%s);" % (
                    outer_scope_cname,
1855
                    cenv.scope_class.type.empty_declaration_code(),
1856 1857 1858 1859
                    Naming.self_cname))
            else:
                code.putln("%s = (%s) %s;" % (
                    outer_scope_cname,
1860
                    cenv.scope_class.type.empty_declaration_code(),
1861
                    Naming.self_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1862
            if lenv.is_passthrough:
Stefan Behnel's avatar
Stefan Behnel committed
1863
                code.putln("%s = %s;" % (Naming.cur_scope_cname, outer_scope_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1864
            elif self.needs_closure:
1865
                # inner closures own a reference to their outer parent
1866
                code.put_incref(outer_scope_cname, cenv.scope_class.type)
1867
                code.put_giveref(outer_scope_cname)
1868
        # ----- Trace function call
1869
        if profile or linetrace:
1870 1871
            # this looks a bit late, but if we don't get here due to a
            # fatal error before hand, it's not really worth tracing
1872 1873 1874 1875 1876 1877 1878 1879
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                if self.is_wrapper:
                    trace_name = self.entry.name + " (wrapper)"
                else:
                    trace_name = self.entry.name
                code.put_trace_call(
                    trace_name, self.pos, nogil=not code.funcstate.gil_owned)
1880
            code.funcstate.can_trace = True
William Stein's avatar
William Stein committed
1881
        # ----- Fetch arguments
1882
        self.generate_argument_parsing_code(env, code)
1883
        # If an argument is assigned to in the body, we must
Robert Bradshaw's avatar
Robert Bradshaw committed
1884
        # incref it to properly keep track of refcounts.
1885
        is_cdef = isinstance(self, CFuncDefNode)
Robert Bradshaw's avatar
Robert Bradshaw committed
1886
        for entry in lenv.arg_entries:
1887
            if entry.type.is_pyobject:
1888
                if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
1889
                    code.put_var_incref(entry)
1890

1891
            # Note: defaults are always incref-ed. For def functions, we
1892 1893 1894
            #       we aquire arguments from object converstion, so we have
            #       new references. If we are a cdef function, we need to
            #       incref our arguments
1895
            elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1:
1896
                code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
1897
        for entry in lenv.var_entries:
1898
            if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure:
1899 1900 1901 1902
                if entry.xdecref_cleanup:
                    code.put_var_xincref(entry)
                else:
                    code.put_var_incref(entry)
1903

1904 1905
        # ----- Initialise local buffer auxiliary variables
        for entry in lenv.var_entries + lenv.arg_entries:
1906 1907
            if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
                Buffer.put_init_vars(entry, code)
1908

1909
        # ----- Check and convert arguments
William Stein's avatar
William Stein committed
1910
        self.generate_argument_type_tests(code)
1911 1912 1913
        # ----- Acquire buffer arguments
        for entry in lenv.arg_entries:
            if entry.type.is_buffer:
1914 1915
                Buffer.put_acquire_arg_buffer(entry, code, self.pos)

1916 1917
        if acquire_gil_for_var_decls_only:
            code.put_release_ensured_gil()
1918
            code.funcstate.gil_owned = False
1919

1920 1921 1922
        # -------------------------
        # ----- Function body -----
        # -------------------------
1923
        self.generate_function_body(env, code)
1924

1925
        code.mark_pos(self.pos, trace=False)
William Stein's avatar
William Stein committed
1926
        code.putln("")
1927 1928 1929
        code.putln("/* function exit code */")

        # ----- Default return value
1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940
        if not self.body.is_terminator:
            if self.return_type.is_pyobject:
                #if self.return_type.is_extension_type:
                #    lhs = "(PyObject *)%s" % Naming.retval_cname
                #else:
                lhs = Naming.retval_cname
                code.put_init_to_py_none(lhs, self.return_type)
            else:
                val = self.return_type.default_value
                if val:
                    code.putln("%s = %s;" % (Naming.retval_cname, val))
1941 1942
                elif not self.return_type.is_void:
                    code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
William Stein's avatar
William Stein committed
1943
        # ----- Error cleanup
1944
        if code.error_label in code.labels_used:
1945 1946
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
1947
            code.put_label(code.error_label)
1948
            for cname, type in code.funcstate.all_managed_temps():
1949
                code.put_xdecref(cname, type, have_gil=not lenv.nogil)
1950 1951 1952

            # Clean up buffers -- this calls a Python function
            # so need to save and restore error state
1953
            buffers_present = len(used_buffer_entries) > 0
1954
            #memslice_entries = [e for e in lenv.entries.values() if e.type.is_memoryviewslice]
1955
            if buffers_present:
1956
                code.globalstate.use_utility_code(restore_exception_utility_code)
1957
                code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
1958 1959
                code.putln("__Pyx_PyThreadState_declare")
                code.putln("__Pyx_PyThreadState_assign")
1960
                code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);")
1961 1962
                for entry in used_buffer_entries:
                    Buffer.put_release_buffer_code(code, entry)
1963
                    #code.putln("%s = 0;" % entry.cname)
1964
                code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
1965

1966 1967 1968 1969 1970 1971
            if self.return_type.is_memoryviewslice:
                MemoryView.put_init_entry(Naming.retval_cname, code)
                err_val = Naming.retval_cname
            else:
                err_val = self.error_value()

1972 1973
            exc_check = self.caller_will_check_exceptions()
            if err_val is not None or exc_check:
1974
                # TODO: Fix exception tracing (though currently unused by cProfile).
Robert Bradshaw's avatar
Robert Bradshaw committed
1975 1976
                # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                # code.put_trace_exception()
1977

Mark Florisson's avatar
Mark Florisson committed
1978
                if lenv.nogil and not lenv.has_with_gil_block:
1979 1980 1981
                    code.putln("{")
                    code.put_ensure_gil()

1982
                code.put_add_traceback(self.entry.qualified_name)
1983

Mark Florisson's avatar
Mark Florisson committed
1984
                if lenv.nogil and not lenv.has_with_gil_block:
1985 1986
                    code.put_release_ensured_gil()
                    code.putln("}")
1987
            else:
1988 1989 1990
                warning(self.entry.pos,
                        "Unraisable exception in function '%s'." %
                        self.entry.qualified_name, 0)
1991
                code.put_unraisable(self.entry.qualified_name, lenv.nogil)
1992 1993 1994 1995
            default_retval = self.return_type.default_value
            if err_val is None and default_retval:
                err_val = default_retval
            if err_val is not None:
1996
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
1997 1998
            elif not self.return_type.is_void:
                code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
1999 2000 2001 2002 2003 2004 2005

            if is_getbuffer_slot:
                self.getbuffer_error_cleanup(code)

            # If we are using the non-error cleanup section we should
            # jump past it if we have an error. The if-test below determine
            # whether this section is used.
2006
            if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice:
2007 2008 2009
                code.put_goto(code.return_from_error_cleanup_label)

        # ----- Non-error return cleanup
William Stein's avatar
William Stein committed
2010
        code.put_label(code.return_label)
2011 2012
        for entry in used_buffer_entries:
            Buffer.put_release_buffer_code(code, entry)
2013 2014
        if is_getbuffer_slot:
            self.getbuffer_normal_cleanup(code)
2015 2016 2017

        if self.return_type.is_memoryviewslice:
            # See if our return value is uninitialized on non-error return
2018
            # from . import MemoryView
2019
            # MemoryView.err_if_nogil_initialized_check(self.pos, env)
2020
            cond = code.unlikely(self.return_type.error_condition(Naming.retval_cname))
2021 2022 2023 2024 2025
            code.putln(
                'if (%s) {' % cond)
            if env.nogil:
                code.put_ensure_gil()
            code.putln(
2026
                'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");')
2027 2028 2029 2030 2031
            if env.nogil:
                code.put_release_ensured_gil()
            code.putln(
                '}')

2032 2033
        # ----- Return cleanup for both error and no-error return
        code.put_label(code.return_from_error_cleanup_label)
2034

2035
        for entry in lenv.var_entries:
2036 2037
            if not entry.used or entry.in_closure:
                continue
2038

2039
            if entry.type.is_memoryviewslice:
2040
                code.put_xdecref_memoryviewslice(entry.cname, have_gil=not lenv.nogil)
2041
            elif entry.type.is_pyobject:
2042
                if not entry.is_arg or len(entry.cf_assignments) > 1:
2043 2044 2045 2046
                    if entry.xdecref_cleanup:
                        code.put_var_xdecref(entry)
                    else:
                        code.put_var_decref(entry)
2047

Robert Bradshaw's avatar
Robert Bradshaw committed
2048
        # Decref any increfed args
2049
        for entry in lenv.arg_entries:
2050
            if entry.type.is_pyobject:
2051
                if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
2052
                    code.put_var_decref(entry)
2053 2054 2055 2056
            elif (entry.type.is_memoryviewslice and
                  (not is_cdef or len(entry.cf_assignments) > 1)):
                # decref slices of def functions and acquired slices from cdef
                # functions, but not borrowed slices from cdef functions.
2057 2058
                code.put_xdecref_memoryviewslice(entry.cname,
                                                 have_gil=not lenv.nogil)
Robert Bradshaw's avatar
Robert Bradshaw committed
2059 2060
        if self.needs_closure:
            code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
2061

2062
        # ----- Return
2063
        # This code is duplicated in ModuleNode.generate_module_init_func
2064 2065 2066 2067
        if not lenv.nogil:
            default_retval = self.return_type.default_value
            err_val = self.error_value()
            if err_val is None and default_retval:
Stefan Behnel's avatar
Stefan Behnel committed
2068
                err_val = default_retval  # FIXME: why is err_val not used?
2069 2070
            if self.return_type.is_pyobject:
                code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
2071

2072 2073
        if self.entry.is_special and self.entry.name == "__hash__":
            # Returning -1 for __hash__ is supposed to signal an error
2074
            # We do as Python instances and coerce -1 into -2.
2075
            code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % (
2076
                Naming.retval_cname, Naming.retval_cname))
2077

2078 2079
        if profile or linetrace:
            code.funcstate.can_trace = False
2080 2081 2082 2083 2084 2085 2086 2087
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                if self.return_type.is_pyobject:
                    code.put_trace_return(
                        Naming.retval_cname, nogil=not code.funcstate.gil_owned)
                else:
                    code.put_trace_return(
                        "Py_None", nogil=not code.funcstate.gil_owned)
2088

2089
        if not lenv.nogil:
Stefan Behnel's avatar
Stefan Behnel committed
2090
            # GIL holding function
2091
            code.put_finish_refcount_context()
2092

2093 2094
        if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
            # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
2095
            code.put_release_ensured_gil()
2096
            code.funcstate.gil_owned = False
2097

William Stein's avatar
William Stein committed
2098
        if not self.return_type.is_void:
2099
            code.putln("return %s;" % Naming.retval_cname)
2100

William Stein's avatar
William Stein committed
2101
        code.putln("}")
2102 2103 2104 2105

        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

2106
        # ----- Go back and insert temp variable declarations
2107
        tempvardecl_code.put_temp_declarations(code.funcstate)
2108

2109
        # ----- Python version
2110
        code.exit_cfunc_scope()
2111
        if self.py_func:
2112
            self.py_func.generate_function_definitions(env, code)
2113
        self.generate_wrapper_functions(code)
William Stein's avatar
William Stein committed
2114 2115 2116 2117

    def declare_argument(self, env, arg):
        if arg.type.is_void:
            error(arg.pos, "Invalid use of 'void'")
2118
        elif not arg.type.is_complete() and not (arg.type.is_array or arg.type.is_memoryviewslice):
2119
            error(arg.pos, "Argument type '%s' is incomplete" % arg.type)
William Stein's avatar
William Stein committed
2120
        return env.declare_arg(arg.name, arg.type, arg.pos)
2121

2122 2123 2124
    def generate_arg_type_test(self, arg, code):
        # Generate type test for one argument.
        if arg.type.typeobj_is_available():
2125 2126
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ArgTypeTest", "FunctionArguments.c"))
2127 2128 2129 2130
            typeptr_cname = arg.type.typeptr_cname
            arg_code = "((PyObject *)%s)" % arg.entry.cname
            code.putln(
                'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % (
2131
                    arg_code,
2132 2133 2134
                    typeptr_cname,
                    arg.accept_none,
                    arg.name,
2135
                    arg.type.is_builtin_type and arg.type.require_exact,
2136 2137
                    code.error_goto(arg.pos)))
        else:
2138
            error(arg.pos, "Cannot test type of extern C class without type object name specification")
2139 2140 2141

    def generate_arg_none_check(self, arg, code):
        # Generate None check for one argument.
2142 2143 2144 2145 2146 2147
        if arg.type.is_memoryviewslice:
            cname = "%s.memview" % arg.entry.cname
        else:
            cname = arg.entry.cname

        code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname)
2148 2149
        code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % (
            max(200, len(arg.name)), arg.name,
2150 2151
            code.error_goto(arg.pos)))
        code.putln('}')
2152

2153
    def generate_wrapper_functions(self, code):
William Stein's avatar
William Stein committed
2154 2155 2156
        pass

    def generate_execution_code(self, code):
2157
        code.mark_pos(self.pos)
2158 2159
        # Evaluate and store argument default values
        for arg in self.args:
2160 2161
            if not arg.is_dynamic:
                arg.generate_assignment_code(code)
William Stein's avatar
William Stein committed
2162

2163
    #
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2164
    # Special code for the __getbuffer__ function
2165 2166 2167 2168 2169
    #
    def getbuffer_init(self, code):
        info = self.local_scope.arg_entries[1].cname
        # Python 3.0 betas have a bug in memoryview which makes it call
        # getbuffer with a NULL parameter. For now we work around this;
2170 2171
        # the following block should be removed when this bug is fixed.
        code.putln("if (%s != NULL) {" % info)
2172
        code.putln("%s->obj = Py_None; __Pyx_INCREF(Py_None);" % info)
2173
        code.put_giveref("%s->obj" % info) # Do not refnanny object within structs
2174
        code.putln("}")
2175 2176 2177

    def getbuffer_error_cleanup(self, code):
        info = self.local_scope.arg_entries[1].cname
2178 2179
        code.putln("if (%s != NULL && %s->obj != NULL) {"
                   % (info, info))
2180
        code.put_gotref("%s->obj" % info)
2181 2182 2183
        code.putln("__Pyx_DECREF(%s->obj); %s->obj = NULL;"
                   % (info, info))
        code.putln("}")
2184 2185 2186

    def getbuffer_normal_cleanup(self, code):
        info = self.local_scope.arg_entries[1].cname
2187
        code.putln("if (%s != NULL && %s->obj == Py_None) {" % (info, info))
2188 2189 2190
        code.put_gotref("Py_None")
        code.putln("__Pyx_DECREF(Py_None); %s->obj = NULL;" % info)
        code.putln("}")
William Stein's avatar
William Stein committed
2191

2192
    def get_preprocessor_guard(self):
2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203
        if not self.entry.is_special:
            return None
        name = self.entry.name
        slot = TypeSlots.method_name_to_slot.get(name)
        if not slot:
            return None
        if name == '__long__' and not self.entry.scope.lookup_here('__int__'):
            return None
        if name in ("__getbuffer__", "__releasebuffer__") and self.entry.scope.is_c_class_scope:
            return None
        return slot.preprocessor_guard_code()
2204 2205


William Stein's avatar
William Stein committed
2206 2207 2208
class CFuncDefNode(FuncDefNode):
    #  C function definition.
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
2209
    #  modifiers     ['inline']
William Stein's avatar
William Stein committed
2210 2211 2212
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarator    CDeclaratorNode
2213 2214 2215
    #  cfunc_declarator  the CFuncDeclarator of this function
    #                    (this is also available through declarator or a
    #                     base thereof)
William Stein's avatar
William Stein committed
2216
    #  body          StatListNode
2217
    #  api           boolean
2218
    #  decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2219
    #
2220
    #  with_gil      boolean    Acquire GIL around body
William Stein's avatar
William Stein committed
2221
    #  type          CFuncType
2222
    #  py_func       wrapper for calling from Python
2223
    #  overridable   whether or not this is a cpdef function
2224
    #  inline_in_pxd whether this is an inline function in a pxd file
2225
    #  template_declaration  String or None   Used for c++ class methods
Robert Bradshaw's avatar
Robert Bradshaw committed
2226
    #  is_const_method whether this is a const method
Robert Bradshaw's avatar
Robert Bradshaw committed
2227
    #  is_static_method whether this is a static method
2228
    #  is_c_class_method whether this is a cclass method
2229

2230
    child_attrs = ["base_type", "declarator", "body", "py_func_stat"]
2231 2232

    inline_in_pxd = False
2233
    decorators = None
2234
    directive_locals = None
2235
    directive_returns = None
2236
    override = None
2237
    template_declaration = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2238
    is_const_method = False
2239
    py_func_stat = None
2240

William Stein's avatar
William Stein committed
2241 2242
    def unqualified_name(self):
        return self.entry.name
2243

2244 2245 2246 2247 2248
    @property
    def code_object(self):
        # share the CodeObject with the cpdef wrapper (if available)
        return self.py_func.code_object if self.py_func else None

William Stein's avatar
William Stein committed
2249
    def analyse_declarations(self, env):
2250
        self.is_c_class_method = env.is_c_class_scope
2251 2252
        if self.directive_locals is None:
            self.directive_locals = {}
2253
        self.directive_locals.update(env.directives['locals'])
2254 2255 2256 2257 2258 2259 2260
        if self.directive_returns is not None:
            base_type = self.directive_returns.analyse_as_type(env)
            if base_type is None:
                error(self.directive_returns.pos, "Not a type")
                base_type = PyrexTypes.error_type
        else:
            base_type = self.base_type.analyse(env)
2261
        self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod')
2262
        # The 2 here is because we need both function and argument names.
2263
        if isinstance(self.declarator, CFuncDeclaratorNode):
2264 2265 2266
            name_declarator, type = self.declarator.analyse(
                base_type, env, nonempty=2 * (self.body is not None),
                directive_locals=self.directive_locals)
2267
        else:
2268 2269
            name_declarator, type = self.declarator.analyse(
                base_type, env, nonempty=2 * (self.body is not None))
2270
        if not type.is_cfunction:
2271
            error(self.pos, "Suite attached to non-function declaration")
William Stein's avatar
William Stein committed
2272 2273 2274 2275 2276
        # Remember the actual type according to the function header
        # written here, because the type in the symbol table entry
        # may be different if we're overriding a C method inherited
        # from the base type of an extension type.
        self.type = type
2277
        type.is_overridable = self.overridable
2278 2279 2280
        declarator = self.declarator
        while not hasattr(declarator, 'args'):
            declarator = declarator.base
2281 2282

        self.cfunc_declarator = declarator
2283
        self.args = declarator.args
2284

2285 2286 2287
        opt_arg_count = self.cfunc_declarator.optional_arg_count
        if (self.visibility == 'public' or self.api) and opt_arg_count:
            error(self.cfunc_declarator.pos,
2288
                  "Function with optional arguments may not be declared public or api")
2289

2290
        if type.exception_check == '+' and self.visibility != 'extern':
2291 2292 2293
            warning(self.cfunc_declarator.pos,
                    "Only extern functions can throw C++ exceptions.")

2294
        for formal_arg, type_arg in zip(self.args, type.args):
2295
            self.align_argument_type(env, type_arg)
2296
            formal_arg.type = type_arg.type
2297
            formal_arg.name = type_arg.name
2298
            formal_arg.cname = type_arg.cname
2299

2300 2301
            self._validate_type_visibility(type_arg.type, type_arg.pos, env)

2302 2303 2304
            if type_arg.type.is_fused:
                self.has_fused_arguments = True

2305 2306
            if type_arg.type.is_buffer and 'inline' in self.modifiers:
                warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
2307

2308
            if type_arg.type.is_buffer or type_arg.type.is_pythran_expr:
2309 2310
                if self.type.nogil:
                    error(formal_arg.pos,
2311
                          "Buffer may not be acquired without the GIL. Consider using memoryview slices instead.")
2312 2313 2314
                elif 'inline' in self.modifiers:
                    warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)

2315 2316
        self._validate_type_visibility(type.return_type, self.pos, env)

William Stein's avatar
William Stein committed
2317 2318
        name = name_declarator.name
        cname = name_declarator.cname
2319

Robert Bradshaw's avatar
Robert Bradshaw committed
2320
        type.is_const_method = self.is_const_method
Robert Bradshaw's avatar
Robert Bradshaw committed
2321
        type.is_static_method = self.is_static_method
William Stein's avatar
William Stein committed
2322
        self.entry = env.declare_cfunction(
2323
            name, type, self.pos,
2324 2325 2326
            cname=cname, visibility=self.visibility, api=self.api,
            defining=self.body is not None, modifiers=self.modifiers,
            overridable=self.overridable)
2327
        self.entry.inline_func_in_pxd = self.inline_in_pxd
William Stein's avatar
William Stein committed
2328
        self.return_type = type.return_type
2329
        if self.return_type.is_array and self.visibility != 'extern':
2330
            error(self.pos, "Function cannot return an array")
2331 2332
        if self.return_type.is_cpp_class:
            self.return_type.check_nullary_constructor(self.pos, "used as a return value")
2333

Robert Bradshaw's avatar
Robert Bradshaw committed
2334
        if self.overridable and not env.is_module_scope and not self.is_static_method:
2335 2336 2337
            if len(self.args) < 1 or not self.args[0].type.is_pyobject:
                # An error will be produced in the cdef function
                self.overridable = False
2338

2339 2340 2341 2342
        self.declare_cpdef_wrapper(env)
        self.create_local_scope(env)

    def declare_cpdef_wrapper(self, env):
2343
        if self.overridable:
2344 2345 2346
            if self.is_static_method:
                # TODO(robertwb): Finish this up, perhaps via more function refactoring.
                error(self.pos, "static cpdef methods not yet supported")
2347
            name = self.entry.name
2348
            py_func_body = self.call_self_node(is_module_scope=env.is_module_scope)
2349 2350 2351 2352 2353 2354
            if self.is_static_method:
                from .ExprNodes import NameNode
                decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))]
                decorators[0].decorator.analyse_types(env)
            else:
                decorators = []
2355 2356 2357 2358 2359 2360 2361 2362 2363
            self.py_func = DefNode(pos=self.pos,
                                   name=self.entry.name,
                                   args=self.args,
                                   star_arg=None,
                                   starstar_arg=None,
                                   doc=self.doc,
                                   body=py_func_body,
                                   decorators=decorators,
                                   is_wrapper=1)
2364
            self.py_func.is_module_scope = env.is_module_scope
2365
            self.py_func.analyse_declarations(env)
2366
            self.py_func.entry.is_overridable = True
2367
            self.py_func_stat = StatListNode(self.pos, stats=[self.py_func])
2368
            self.py_func.type = PyrexTypes.py_object_type
2369
            self.entry.as_variable = self.py_func.entry
2370
            self.entry.used = self.entry.as_variable.used = True
2371 2372
            # Reset scope entry the above cfunction
            env.entries[name] = self.entry
2373
            if (not self.entry.is_final_cmethod and
2374 2375
                    (not env.is_module_scope or Options.lookup_module_cpdef)):
                self.override = OverrideCheckNode(self.pos, py_func=self.py_func)
2376
                self.body = StatListNode(self.pos, stats=[self.override, self.body])
2377

2378 2379 2380 2381 2382 2383 2384 2385 2386 2387
    def _validate_type_visibility(self, type, pos, env):
        """
        Ensure that types used in cdef functions are public or api, or
        defined in a C header.
        """
        public_or_api = (self.visibility == 'public' or self.api)
        entry = getattr(type, 'entry', None)
        if public_or_api and entry and env.is_module_scope:
            if not (entry.visibility in ('public', 'extern') or
                    entry.api or entry.in_cinclude):
2388
                error(pos, "Function declared public or api may not have private types")
2389

2390
    def call_self_node(self, omit_optional_args=0, is_module_scope=0):
2391
        from . import ExprNodes
2392 2393 2394 2395
        args = self.type.args
        if omit_optional_args:
            args = args[:len(args) - self.type.optional_arg_count]
        arg_names = [arg.name for arg in args]
2396
        if is_module_scope:
2397
            cfunc = ExprNodes.NameNode(self.pos, name=self.entry.name)
2398 2399 2400 2401
            call_arg_names = arg_names
            skip_dispatch = Options.lookup_module_cpdef
        elif self.type.is_static_method:
            class_entry = self.entry.scope.parent_type.entry
2402 2403 2404
            class_node = ExprNodes.NameNode(self.pos, name=class_entry.name)
            class_node.entry = class_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=class_node, attribute=self.entry.name)
2405 2406 2407
            # Calling static c(p)def methods on an instance disallowed.
            # TODO(robertwb): Support by passing self to check for override?
            skip_dispatch = True
2408
        else:
2409 2410 2411 2412
            type_entry = self.type.args[0].type.entry
            type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
            type_arg.entry = type_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
2413
        skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
2414 2415 2416 2417 2418
        c_call = ExprNodes.SimpleCallNode(
            self.pos,
            function=cfunc,
            args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names],
            wrapper_call=skip_dispatch)
2419 2420
        return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)

William Stein's avatar
William Stein committed
2421 2422 2423 2424 2425
    def declare_arguments(self, env):
        for arg in self.type.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            self.declare_argument(env, arg)
2426

2427
    def need_gil_acquisition(self, lenv):
2428 2429
        return self.type.with_gil

2430
    def nogil_check(self, env):
2431
        type = self.type
2432
        with_gil = type.with_gil
2433 2434 2435 2436
        if type.nogil and not with_gil:
            if type.return_type.is_pyobject:
                error(self.pos,
                      "Function with Python return type cannot be declared nogil")
2437
            for entry in self.local_scope.var_entries:
2438
                if entry.type.is_pyobject and not entry.in_with_gil_block:
2439 2440
                    error(self.pos, "Function declared nogil has Python locals or temporaries")

2441
    def analyse_expressions(self, env):
2442
        self.local_scope.directives = env.directives
2443 2444 2445 2446
        if self.py_func_stat is not None:
            # this will also analyse the default values and the function name assignment
            self.py_func_stat = self.py_func_stat.analyse_expressions(env)
        elif self.py_func is not None:
2447
            # this will also analyse the default values
2448
            self.py_func = self.py_func.analyse_expressions(env)
2449 2450
        else:
            self.analyse_default_values(env)
2451
            self.analyse_annotations(env)
2452
        self.acquire_gil = self.need_gil_acquisition(self.local_scope)
2453
        return self
2454

Robert Bradshaw's avatar
Robert Bradshaw committed
2455 2456 2457
    def needs_assignment_synthesis(self, env, code=None):
        return False

2458
    def generate_function_header(self, code, with_pymethdef, with_opt_args=1, with_dispatch=1, cname=None):
2459
        scope = self.local_scope
William Stein's avatar
William Stein committed
2460 2461
        arg_decls = []
        type = self.type
2462
        for arg in type.args[:len(type.args)-type.optional_arg_count]:
2463 2464 2465 2466 2467
            arg_decl = arg.declaration_code()
            entry = scope.lookup(arg.name)
            if not entry.cf_used:
                arg_decl = 'CYTHON_UNUSED %s' % arg_decl
            arg_decls.append(arg_decl)
2468
        if with_dispatch and self.overridable:
2469 2470 2471 2472 2473 2474
            dispatch_arg = PyrexTypes.c_int_type.declaration_code(
                Naming.skip_dispatch_cname)
            if self.override:
                arg_decls.append(dispatch_arg)
            else:
                arg_decls.append('CYTHON_UNUSED %s' % dispatch_arg)
2475 2476
        if type.optional_arg_count and with_opt_args:
            arg_decls.append(type.op_arg_struct.declaration_code(Naming.optional_args_cname))
William Stein's avatar
William Stein committed
2477 2478 2479 2480
        if type.has_varargs:
            arg_decls.append("...")
        if not arg_decls:
            arg_decls = ["void"]
2481 2482
        if cname is None:
            cname = self.entry.func_cname
2483
        entity = type.function_header_code(cname, ', '.join(arg_decls))
2484
        if self.entry.visibility == 'private' and '::' not in cname:
2485
            storage_class = "static "
William Stein's avatar
William Stein committed
2486
        else:
2487
            storage_class = ""
2488
        dll_linkage = None
2489
        modifiers = code.build_function_modifiers(self.entry.func_modifiers)
Robert Bradshaw's avatar
Robert Bradshaw committed
2490

2491 2492
        header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage)
        #print (storage_class, modifiers, header)
2493
        needs_proto = self.is_c_class_method
2494
        if self.template_declaration:
2495 2496
            if needs_proto:
                code.globalstate.parts['module_declarations'].putln(self.template_declaration)
2497
            code.putln(self.template_declaration)
2498
        if needs_proto:
2499 2500
            code.globalstate.parts['module_declarations'].putln(
                "%s%s%s; /* proto*/" % (storage_class, modifiers, header))
2501
        code.putln("%s%s%s {" % (storage_class, modifiers, header))
William Stein's avatar
William Stein committed
2502 2503

    def generate_argument_declarations(self, env, code):
2504
        scope = self.local_scope
2505
        for arg in self.args:
2506
            if arg.default:
2507
                entry = scope.lookup(arg.name)
2508
                if self.override or entry.cf_used:
2509 2510 2511
                    result = arg.calculate_default_value_code(code)
                    code.putln('%s = %s;' % (
                        arg.type.declaration_code(arg.cname), result))
2512

William Stein's avatar
William Stein committed
2513 2514
    def generate_keyword_list(self, code):
        pass
2515

2516
    def generate_argument_parsing_code(self, env, code):
2517
        i = 0
2518
        used = 0
2519
        scope = self.local_scope
2520 2521
        if self.type.optional_arg_count:
            code.putln('if (%s) {' % Naming.optional_args_cname)
2522
            for arg in self.args:
2523
                if arg.default:
2524
                    entry = scope.lookup(arg.name)
2525
                    if self.override or entry.cf_used:
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535
                        code.putln('if (%s->%sn > %s) {' %
                                   (Naming.optional_args_cname,
                                    Naming.pyrex_prefix, i))
                        declarator = arg.declarator
                        while not hasattr(declarator, 'name'):
                            declarator = declarator.base
                        code.putln('%s = %s->%s;' %
                                   (arg.cname, Naming.optional_args_cname,
                                    self.type.opt_arg_cname(declarator.name)))
                        used += 1
2536
                    i += 1
2537
            for _ in range(used):
2538
                code.putln('}')
2539
            code.putln('}')
2540

2541 2542 2543 2544 2545 2546 2547 2548 2549 2550
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure and not arg.default:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(scope.lookup_here(arg.name))


William Stein's avatar
William Stein committed
2551 2552
    def generate_argument_conversion_code(self, code):
        pass
2553

William Stein's avatar
William Stein committed
2554
    def generate_argument_type_tests(self, code):
2555 2556 2557 2558 2559
        # Generate type tests for args whose type in a parent
        # class is a supertype of the declared type.
        for arg in self.type.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
2560 2561
            elif arg.type.is_pyobject and not arg.accept_none:
                self.generate_arg_none_check(arg, code)
2562

2563 2564 2565 2566 2567
    def generate_execution_code(self, code):
        super(CFuncDefNode, self).generate_execution_code(code)
        if self.py_func_stat:
            self.py_func_stat.generate_execution_code(code)

William Stein's avatar
William Stein committed
2568 2569 2570 2571
    def error_value(self):
        if self.return_type.is_pyobject:
            return "0"
        else:
2572 2573
            #return None
            return self.entry.type.exception_value
2574

William Stein's avatar
William Stein committed
2575
    def caller_will_check_exceptions(self):
2576
        return self.entry.type.exception_check
2577

2578 2579
    def generate_wrapper_functions(self, code):
        # If the C signature of a function has changed, we need to generate
2580
        # wrappers to put in the slots here.
2581 2582 2583 2584 2585 2586 2587
        k = 0
        entry = self.entry
        func_type = entry.type
        while entry.prev_entry is not None:
            k += 1
            entry = entry.prev_entry
            entry.func_cname = "%s%swrap_%s" % (self.entry.func_cname, Naming.pyrex_prefix, k)
2588
            code.putln()
2589 2590 2591 2592 2593
            self.generate_function_header(
                code, 0,
                with_dispatch=entry.type.is_overridable,
                with_opt_args=entry.type.optional_arg_count,
                cname=entry.func_cname)
2594 2595 2596 2597
            if not self.return_type.is_void:
                code.put('return ')
            args = self.type.args
            arglist = [arg.cname for arg in args[:len(args)-self.type.optional_arg_count]]
2598 2599 2600 2601 2602 2603 2604 2605
            if entry.type.is_overridable:
                arglist.append(Naming.skip_dispatch_cname)
            elif func_type.is_overridable:
                arglist.append('0')
            if entry.type.optional_arg_count:
                arglist.append(Naming.optional_args_cname)
            elif func_type.optional_arg_count:
                arglist.append('NULL')
2606 2607
            code.putln('%s(%s);' % (self.entry.func_cname, ', '.join(arglist)))
            code.putln('}')
2608

William Stein's avatar
William Stein committed
2609 2610 2611 2612 2613

class PyArgDeclNode(Node):
    # Argument which must be a Python object (used
    # for * and ** arguments).
    #
2614 2615 2616
    # name        string
    # entry       Symtab.Entry
    # annotation  ExprNode or None   Py3 argument annotation
2617
    child_attrs = []
2618 2619
    is_self_arg = False
    is_type_arg = False
2620 2621 2622

    def generate_function_definitions(self, env, code):
        self.entry.generate_function_definitions(env, code)
2623

2624

2625 2626 2627
class DecoratorNode(Node):
    # A decorator
    #
2628
    # decorator    NameNode or CallNode or AttributeNode
2629 2630
    child_attrs = ['decorator']

William Stein's avatar
William Stein committed
2631 2632 2633 2634 2635

class DefNode(FuncDefNode):
    # A Python function definition.
    #
    # name          string                 the Python name of the function
Stefan Behnel's avatar
Stefan Behnel committed
2636
    # lambda_name   string                 the internal name of a lambda 'function'
2637
    # decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2638
    # args          [CArgDeclNode]         formal arguments
2639
    # doc           EncodedString or None
William Stein's avatar
William Stein committed
2640
    # body          StatListNode
2641 2642
    # return_type_annotation
    #               ExprNode or None       the Py3 return type annotation
William Stein's avatar
William Stein committed
2643 2644 2645 2646
    #
    #  The following subnode is constructed internally
    #  when the def statement is inside a Python class definition.
    #
2647 2648 2649
    #  fused_py_func        DefNode     The original fused cpdef DefNode
    #                                   (in case this is a specialization)
    #  specialized_cpdefs   [DefNode]   list of specialized cpdef DefNodes
2650
    #  py_cfunc_node  PyCFunctionNode/InnerFunctionNode   The PyCFunction to create and assign
2651 2652
    #
    # decorator_indirection IndirectionNode Used to remove __Pyx_Method_ClassMethod for fused functions
2653

2654
    child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "return_type_annotation"]
2655

2656 2657 2658
    is_staticmethod = False
    is_classmethod = False

Stefan Behnel's avatar
Stefan Behnel committed
2659
    lambda_name = None
2660
    reqd_kw_flags_cname = "0"
2661
    is_wrapper = 0
2662
    no_assignment_synthesis = 0
2663
    decorators = None
2664
    return_type_annotation = None
2665
    entry = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2666
    acquire_gil = 0
2667
    self_in_stararg = 0
2668
    py_cfunc_node = None
2669
    requires_classobj = False
2670
    defaults_struct = None # Dynamic kwrds structure name
2671
    doc = None
2672

2673 2674
    fused_py_func = False
    specialized_cpdefs = None
2675 2676 2677
    py_wrapper = None
    py_wrapper_required = True
    func_cname = None
2678

2679 2680
    defaults_getter = None

2681 2682
    def __init__(self, pos, **kwds):
        FuncDefNode.__init__(self, pos, **kwds)
2683
        k = rk = r = 0
2684 2685
        for arg in self.args:
            if arg.kw_only:
2686
                k += 1
2687
                if not arg.default:
2688 2689 2690 2691 2692 2693
                    rk += 1
            if not arg.default:
                r += 1
        self.num_kwonly_args = k
        self.num_required_kw_args = rk
        self.num_required_args = r
2694

2695
    def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, modifiers=None):
2696 2697 2698 2699
        if self.star_arg:
            error(self.star_arg.pos, "cdef function cannot have star argument")
        if self.starstar_arg:
            error(self.starstar_arg.pos, "cdef function cannot have starstar argument")
2700 2701 2702 2703
        if cfunc is None:
            cfunc_args = []
            for formal_arg in self.args:
                name_declarator, type = formal_arg.analyse(scope, nonempty=1)
2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715
                cfunc_args.append(PyrexTypes.CFuncTypeArg(name=name_declarator.name,
                                                          cname=None,
                                                          type=py_object_type,
                                                          pos=formal_arg.pos))
            cfunc_type = PyrexTypes.CFuncType(return_type=py_object_type,
                                              args=cfunc_args,
                                              has_varargs=False,
                                              exception_value=None,
                                              exception_check=False,
                                              nogil=False,
                                              with_gil=False,
                                              is_overridable=overridable)
2716
            cfunc = CVarDefNode(self.pos, type=cfunc_type)
2717
        else:
2718 2719
            if scope is None:
                scope = cfunc.scope
2720 2721 2722
            cfunc_type = cfunc.type
            if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs:
                error(self.pos, "wrong number of arguments")
Stefan Behnel's avatar
Stefan Behnel committed
2723
                error(cfunc.pos, "previous declaration here")
2724 2725
            for i, (formal_arg, type_arg) in enumerate(zip(self.args, cfunc_type.args)):
                name_declarator, type = formal_arg.analyse(scope, nonempty=1,
2726
                                                           is_self_arg=(i == 0 and scope.is_c_class_scope))
Stefan Behnel's avatar
Stefan Behnel committed
2727
                if type is None or type is PyrexTypes.py_object_type:
2728 2729
                    formal_arg.type = type_arg.type
                    formal_arg.name_declarator = name_declarator
2730
        from . import ExprNodes
2731
        if cfunc_type.exception_value is None:
2732 2733
            exception_value = None
        else:
2734 2735
            exception_value = ExprNodes.ConstNode(
                self.pos, value=cfunc_type.exception_value, type=cfunc_type.return_type)
2736
        declarator = CFuncDeclaratorNode(self.pos,
2737 2738 2739 2740 2741 2742 2743
                                         base=CNameDeclaratorNode(self.pos, name=self.name, cname=None),
                                         args=self.args,
                                         has_varargs=False,
                                         exception_check=cfunc_type.exception_check,
                                         exception_value=exception_value,
                                         with_gil=cfunc_type.with_gil,
                                         nogil=cfunc_type.nogil)
2744
        return CFuncDefNode(self.pos,
2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
                            modifiers=modifiers or [],
                            base_type=CAnalysedBaseTypeNode(self.pos, type=cfunc_type.return_type),
                            declarator=declarator,
                            body=self.body,
                            doc=self.doc,
                            overridable=cfunc_type.is_overridable,
                            type=cfunc_type,
                            with_gil=cfunc_type.with_gil,
                            nogil=cfunc_type.nogil,
                            visibility='private',
                            api=False,
                            directive_locals=getattr(cfunc, 'directive_locals', {}),
                            directive_returns=returns)
2758

2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769
    def is_cdef_func_compatible(self):
        """Determines if the function's signature is compatible with a
        cdef function.  This can be used before calling
        .as_cfunction() to see if that will be successful.
        """
        if self.needs_closure:
            return False
        if self.star_arg or self.starstar_arg:
            return False
        return True

William Stein's avatar
William Stein committed
2770
    def analyse_declarations(self, env):
2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
        if self.decorators:
            for decorator in self.decorators:
                func = decorator.decorator
                if func.is_name:
                    self.is_classmethod |= func.name == 'classmethod'
                    self.is_staticmethod |= func.name == 'staticmethod'

        if self.is_classmethod and env.lookup_here('classmethod'):
            # classmethod() was overridden - not much we can do here ...
            self.is_classmethod = False
        if self.is_staticmethod and env.lookup_here('staticmethod'):
            # staticmethod() was overridden - not much we can do here ...
            self.is_staticmethod = False

2785
        if self.name == '__new__' and env.is_py_class_scope:
Vitja Makarov's avatar
Vitja Makarov committed
2786 2787
            self.is_staticmethod = 1

2788
        self.analyse_argument_types(env)
2789 2790 2791 2792
        if self.name == '<lambda>':
            self.declare_lambda_function(env)
        else:
            self.declare_pyfunction(env)
2793

2794 2795
        self.analyse_signature(env)
        self.return_type = self.entry.signature.return_type()
2796 2797 2798 2799 2800 2801 2802
        # if a signature annotation provides a more specific return object type, use it
        if self.return_type is py_object_type and self.return_type_annotation:
            if env.directives['annotation_typing'] and not self.entry.is_special:
                _, return_type = _analyse_signature_annotation(self.return_type_annotation, env)
                if return_type and return_type.is_pyobject:
                    self.return_type = return_type

2803 2804
        self.create_local_scope(env)

2805 2806 2807 2808 2809 2810 2811 2812 2813 2814
        self.py_wrapper = DefNodeWrapper(
            self.pos,
            target=self,
            name=self.entry.name,
            args=self.args,
            star_arg=self.star_arg,
            starstar_arg=self.starstar_arg,
            return_type=self.return_type)
        self.py_wrapper.analyse_declarations(env)

2815
    def analyse_argument_types(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
2816
        self.directive_locals = env.directives['locals']
2817
        allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
2818 2819 2820 2821

        f2s = env.fused_to_specific
        env.fused_to_specific = None

William Stein's avatar
William Stein committed
2822
        for arg in self.args:
2823 2824 2825 2826
            if hasattr(arg, 'name'):
                name_declarator = None
            else:
                base_type = arg.base_type.analyse(env)
2827 2828 2829 2830 2831 2832 2833
                # If we hare in pythran mode and we got a buffer supported by
                # Pythran, we change this node to a fused type
                if has_np_pythran(env) and base_type.is_pythran_expr:
                    base_type = PyrexTypes.FusedType([
                        base_type,
                        #PyrexTypes.PythranExpr(pythran_type(self.type, "numpy_texpr")),
                        base_type.org_buffer])
2834 2835 2836
                name_declarator, type = \
                    arg.declarator.analyse(base_type, env)
                arg.name = name_declarator.name
2837
                arg.type = type
2838 2839 2840 2841

                if type.is_fused:
                    self.has_fused_arguments = True

2842
            self.align_argument_type(env, arg)
2843
            if name_declarator and name_declarator.cname:
2844
                error(self.pos, "Python function argument cannot have C name specification")
2845
            arg.type = arg.type.as_argument_type()
William Stein's avatar
William Stein committed
2846 2847 2848 2849
            arg.hdr_type = None
            arg.needs_conversion = 0
            arg.needs_type_test = 0
            arg.is_generic = 1
2850
            if arg.type.is_pyobject or arg.type.is_buffer or arg.type.is_memoryviewslice:
2851 2852 2853 2854
                if arg.or_none:
                    arg.accept_none = True
                elif arg.not_none:
                    arg.accept_none = False
2855
                elif (arg.type.is_extension_type or arg.type.is_builtin_type
2856
                        or arg.type.is_buffer or arg.type.is_memoryviewslice):
2857 2858 2859 2860 2861 2862
                    if arg.default and arg.default.constant_result is None:
                        # special case: def func(MyType obj = None)
                        arg.accept_none = True
                    else:
                        # default depends on compiler directive
                        arg.accept_none = allow_none_for_extension_args
2863 2864 2865
                else:
                    # probably just a plain 'object'
                    arg.accept_none = True
2866
            else:
2867
                arg.accept_none = True # won't be used, but must be there
2868
                if arg.not_none:
2869
                    error(arg.pos, "Only Python type arguments can have 'not None'")
2870
                if arg.or_none:
2871
                    error(arg.pos, "Only Python type arguments can have 'or None'")
2872 2873
        env.fused_to_specific = f2s

2874 2875 2876 2877 2878
        if has_np_pythran(env):
            self.np_args_idx = [i for i,a in enumerate(self.args) if a.type.is_numpy_buffer]
        else:
            self.np_args_idx = []

William Stein's avatar
William Stein committed
2879
    def analyse_signature(self, env):
2880
        if self.entry.is_special:
2881
            if self.decorators:
2882
                error(self.pos, "special functions of cdef classes cannot have decorators")
2883 2884 2885 2886
            self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg)
        elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg):
            # Use the simpler calling signature for zero- and one-argument functions.
            if self.entry.signature is TypeSlots.pyfunction_signature:
2887 2888
                if len(self.args) == 0:
                    self.entry.signature = TypeSlots.pyfunction_noargs
2889 2890 2891
                elif len(self.args) == 1:
                    if self.args[0].default is None and not self.args[0].kw_only:
                        self.entry.signature = TypeSlots.pyfunction_onearg
2892 2893 2894
            elif self.entry.signature is TypeSlots.pymethod_signature:
                if len(self.args) == 1:
                    self.entry.signature = TypeSlots.unaryfunc
2895 2896 2897
                elif len(self.args) == 2:
                    if self.args[1].default is None and not self.args[1].kw_only:
                        self.entry.signature = TypeSlots.ibinaryfunc
2898

William Stein's avatar
William Stein committed
2899 2900
        sig = self.entry.signature
        nfixed = sig.num_fixed_args()
2901 2902
        if (sig is TypeSlots.pymethod_signature and nfixed == 1
               and len(self.args) == 0 and self.star_arg):
2903 2904 2905 2906 2907 2908 2909
            # this is the only case where a diverging number of
            # arguments is not an error - when we have no explicit
            # 'self' parameter as in method(*args)
            sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used
            self.self_in_stararg = 1
            nfixed = 0

2910 2911
        if self.is_staticmethod and env.is_c_class_scope:
            nfixed = 0
2912
            self.self_in_stararg = True  # FIXME: why for staticmethods?
2913 2914 2915 2916 2917 2918

            self.entry.signature = sig = copy.copy(sig)
            sig.fixed_arg_format = "*"
            sig.is_staticmethod = True
            sig.has_generic_args = True

2919
        if ((self.is_classmethod or self.is_staticmethod) and
2920
                self.has_fused_arguments and env.is_c_class_scope):
2921 2922
            del self.decorator_indirection.stats[:]

2923 2924 2925 2926 2927 2928 2929
        for i in range(min(nfixed, len(self.args))):
            arg = self.args[i]
            arg.is_generic = 0
            if sig.is_self_arg(i) and not self.is_staticmethod:
                if self.is_classmethod:
                    arg.is_type_arg = 1
                    arg.hdr_type = arg.type = Builtin.type_type
William Stein's avatar
William Stein committed
2930
                else:
2931 2932 2933
                    arg.is_self_arg = 1
                    arg.hdr_type = arg.type = env.parent_type
                arg.needs_conversion = 0
William Stein's avatar
William Stein committed
2934
            else:
2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949
                arg.hdr_type = sig.fixed_arg_type(i)
                if not arg.type.same_as(arg.hdr_type):
                    if arg.hdr_type.is_pyobject and arg.type.is_pyobject:
                        arg.needs_type_test = 1
                    else:
                        arg.needs_conversion = 1
            if arg.needs_conversion:
                arg.hdr_cname = Naming.arg_prefix + arg.name
            else:
                arg.hdr_cname = Naming.var_prefix + arg.name

        if nfixed > len(self.args):
            self.bad_signature()
            return
        elif nfixed < len(self.args):
William Stein's avatar
William Stein committed
2950 2951 2952
            if not sig.has_generic_args:
                self.bad_signature()
            for arg in self.args:
2953
                if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type):
William Stein's avatar
William Stein committed
2954
                    arg.needs_type_test = 1
2955

William Stein's avatar
William Stein committed
2956 2957 2958 2959
    def bad_signature(self):
        sig = self.entry.signature
        expected_str = "%d" % sig.num_fixed_args()
        if sig.has_generic_args:
Stefan Behnel's avatar
Stefan Behnel committed
2960
            expected_str += " or more"
William Stein's avatar
William Stein committed
2961 2962 2963 2964 2965
        name = self.name
        if name.startswith("__") and name.endswith("__"):
            desc = "Special method"
        else:
            desc = "Method"
2966 2967
        error(self.pos, "%s %s has wrong number of arguments (%d declared, %s expected)" % (
            desc, self.name, len(self.args), expected_str))
2968

William Stein's avatar
William Stein committed
2969
    def declare_pyfunction(self, env):
2970 2971
        #print "DefNode.declare_pyfunction:", self.name, "in", env ###
        name = self.name
Stefan Behnel's avatar
Stefan Behnel committed
2972
        entry = env.lookup_here(name)
2973 2974
        if entry:
            if entry.is_final_cmethod and not env.parent_type.is_final_type:
Stefan Behnel's avatar
Stefan Behnel committed
2975
                error(self.pos, "Only final types can have final Python (def/cpdef) methods")
2976
            if entry.type.is_cfunction and not entry.is_builtin_cmethod and not self.is_wrapper:
2977
                warning(self.pos, "Overriding cdef method with def method.", 5)
2978
        entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper)
2979
        self.entry = entry
2980
        prefix = env.next_id(env.scope_prefix)
2981
        self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name
2982 2983
        if Options.docstrings:
            entry.doc = embed_position(self.pos, self.doc)
2984
            entry.doc_cname = Naming.funcdoc_prefix + prefix + name
2985
            if entry.is_special:
2986 2987
                if entry.name in TypeSlots.invisible or not entry.doc or (
                        entry.name in '__getattr__' and env.directives['fast_getattr']):
2988 2989 2990
                    entry.wrapperbase_cname = None
                else:
                    entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name
2991 2992
        else:
            entry.doc = None
2993

Stefan Behnel's avatar
Stefan Behnel committed
2994
    def declare_lambda_function(self, env):
2995
        entry = env.declare_lambda_function(self.lambda_name, self.pos)
Stefan Behnel's avatar
Stefan Behnel committed
2996 2997
        entry.doc = None
        self.entry = entry
2998
        self.entry.pyfunc_cname = entry.cname
Stefan Behnel's avatar
Stefan Behnel committed
2999

William Stein's avatar
William Stein committed
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
    def declare_arguments(self, env):
        for arg in self.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            if arg.needs_conversion:
                arg.entry = env.declare_var(arg.name, arg.type, arg.pos)
                if arg.type.is_pyobject:
                    arg.entry.init = "0"
            else:
                arg.entry = self.declare_argument(env, arg)
3010
            arg.entry.is_arg = 1
3011
            arg.entry.used = 1
William Stein's avatar
William Stein committed
3012 3013 3014 3015 3016 3017
            arg.entry.is_self_arg = arg.is_self_arg
        self.declare_python_arg(env, self.star_arg)
        self.declare_python_arg(env, self.starstar_arg)

    def declare_python_arg(self, env, arg):
        if arg:
3018
            if env.directives['infer_types'] != False:
3019 3020 3021 3022
                type = PyrexTypes.unspecified_type
            else:
                type = py_object_type
            entry = env.declare_var(arg.name, type, arg.pos)
3023
            entry.is_arg = 1
3024 3025 3026 3027
            entry.used = 1
            entry.init = "0"
            entry.xdecref_cleanup = 1
            arg.entry = entry
3028

William Stein's avatar
William Stein committed
3029
    def analyse_expressions(self, env):
3030
        self.local_scope.directives = env.directives
William Stein's avatar
William Stein committed
3031
        self.analyse_default_values(env)
3032 3033 3034
        self.analyse_annotations(env)
        if self.return_type_annotation:
            self.return_type_annotation = self.return_type_annotation.analyse_types(env)
3035

3036
        if not self.needs_assignment_synthesis(env) and self.decorators:
3037
            for decorator in self.decorators[::-1]:
3038
                decorator.decorator = decorator.decorator.analyse_expressions(env)
3039

3040
        self.py_wrapper.prepare_argument_coercion(env)
3041
        return self
3042

3043
    def needs_assignment_synthesis(self, env, code=None):
3044
        if self.is_staticmethod:
3045
            return True
3046
        if self.specialized_cpdefs or self.entry.is_fused_specialized:
3047
            return False
3048 3049
        if self.no_assignment_synthesis:
            return False
3050 3051
        if self.entry.is_special:
            return False
3052
        if self.entry.is_anonymous:
3053
            return True
3054
        if env.is_module_scope or env.is_c_class_scope:
3055
            if code is None:
3056
                return self.local_scope.directives['binding']
3057 3058 3059 3060
            else:
                return code.globalstate.directives['binding']
        return env.is_py_class_scope or env.is_closure_scope

3061 3062 3063 3064
    def error_value(self):
        return self.entry.signature.error_value

    def caller_will_check_exceptions(self):
3065
        return self.entry.signature.exception_check
3066 3067

    def generate_function_definitions(self, env, code):
3068
        if self.defaults_getter:
3069 3070
            # defaults getter must never live in class scopes, it's always a module function
            self.defaults_getter.generate_function_definitions(env.global_scope(), code)
3071

3072 3073 3074 3075 3076 3077 3078
        # Before closure cnames are mangled
        if self.py_wrapper_required:
            # func_cname might be modified by @cname
            self.py_wrapper.func_cname = self.entry.func_cname
            self.py_wrapper.generate_function_definitions(env, code)
        FuncDefNode.generate_function_definitions(self, env, code)

3079
    def generate_function_header(self, code, with_pymethdef, proto_only=0):
3080 3081 3082 3083 3084
        if proto_only:
            if self.py_wrapper_required:
                self.py_wrapper.generate_function_header(
                    code, with_pymethdef, True)
            return
William Stein's avatar
William Stein committed
3085
        arg_code_list = []
3086
        if self.entry.signature.has_dummy_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3087 3088 3089
            self_arg = 'PyObject *%s' % Naming.self_cname
            if not self.needs_outer_scope:
                self_arg = 'CYTHON_UNUSED ' + self_arg
3090 3091 3092 3093 3094 3095 3096 3097
            arg_code_list.append(self_arg)

        def arg_decl_code(arg):
            entry = arg.entry
            if entry.in_closure:
                cname = entry.original_cname
            else:
                cname = entry.cname
3098
            decl = entry.type.declaration_code(cname)
Stefan Behnel's avatar
Stefan Behnel committed
3099 3100 3101
            if not entry.cf_used:
                decl = 'CYTHON_UNUSED ' + decl
            return decl
3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167

        for arg in self.args:
            arg_code_list.append(arg_decl_code(arg))
        if self.star_arg:
            arg_code_list.append(arg_decl_code(self.star_arg))
        if self.starstar_arg:
            arg_code_list.append(arg_decl_code(self.starstar_arg))
        arg_code = ', '.join(arg_code_list)
        dc = self.return_type.declaration_code(self.entry.pyfunc_cname)

        decls_code = code.globalstate['decls']
        preprocessor_guard = self.get_preprocessor_guard()
        if preprocessor_guard:
            decls_code.putln(preprocessor_guard)
        decls_code.putln(
            "static %s(%s); /* proto */" % (dc, arg_code))
        if preprocessor_guard:
            decls_code.putln("#endif")
        code.putln("static %s(%s) {" % (dc, arg_code))

    def generate_argument_declarations(self, env, code):
        pass

    def generate_keyword_list(self, code):
        pass

    def generate_argument_parsing_code(self, env, code):
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(arg.entry)
        for arg in self.star_arg, self.starstar_arg:
            if arg:
                put_into_closure(arg.entry)

    def generate_argument_type_tests(self, code):
        pass


class DefNodeWrapper(FuncDefNode):
    # DefNode python wrapper code generator

    defnode = None
    target = None # Target DefNode

    def __init__(self, *args, **kwargs):
        FuncDefNode.__init__(self, *args, **kwargs)
        self.num_kwonly_args = self.target.num_kwonly_args
        self.num_required_kw_args = self.target.num_required_kw_args
        self.num_required_args = self.target.num_required_args
        self.self_in_stararg = self.target.self_in_stararg
        self.signature = None

    def analyse_declarations(self, env):
        target_entry = self.target.entry
        name = self.name
        prefix = env.next_id(env.scope_prefix)
        target_entry.func_cname = Naming.pywrap_prefix + prefix + name
        target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name

        self.signature = target_entry.signature

3168 3169
        self.np_args_idx = self.target.np_args_idx

3170
    def prepare_argument_coercion(self, env):
3171 3172 3173 3174 3175 3176 3177 3178
        # This is only really required for Cython utility code at this time,
        # everything else can be done during code generation.  But we expand
        # all utility code here, simply because we cannot easily distinguish
        # different code types.
        for arg in self.args:
            if not arg.type.is_pyobject:
                if not arg.type.create_from_py_utility_code(env):
                    pass # will fail later
Robert Bradshaw's avatar
Robert Bradshaw committed
3179
            elif arg.hdr_type and not arg.hdr_type.is_pyobject:
3180 3181
                if not arg.hdr_type.create_to_py_utility_code(env):
                    pass # will fail later
3182

3183 3184 3185 3186 3187 3188 3189 3190 3191
        if self.starstar_arg and not self.starstar_arg.entry.cf_used:
            # we will set the kwargs argument to NULL instead of a new dict
            # and must therefore correct the control flow state
            entry = self.starstar_arg.entry
            entry.xdecref_cleanup = 1
            for ass in entry.cf_assignments:
                if not ass.is_arg and ass.lhs.is_name:
                    ass.lhs.cf_maybe_null = True

3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207
    def signature_has_nongeneric_args(self):
        argcount = len(self.args)
        if argcount == 0 or (
                argcount == 1 and (self.args[0].is_self_arg or
                                   self.args[0].is_type_arg)):
            return 0
        return 1

    def signature_has_generic_args(self):
        return self.signature.has_generic_args

    def generate_function_body(self, code):
        args = []
        if self.signature.has_dummy_arg:
            args.append(Naming.self_cname)
        for arg in self.args:
3208 3209 3210
            if arg.hdr_type and not (arg.type.is_memoryviewslice or
                                     arg.type.is_struct or
                                     arg.type.is_complex):
3211 3212 3213
                args.append(arg.type.cast_code(arg.entry.cname))
            else:
                args.append(arg.entry.cname)
3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226
        if self.star_arg:
            args.append(self.star_arg.entry.cname)
        if self.starstar_arg:
            args.append(self.starstar_arg.entry.cname)
        args = ', '.join(args)
        if not self.return_type.is_void:
            code.put('%s = ' % Naming.retval_cname)
        code.putln('%s(%s);' % (
            self.target.entry.pyfunc_cname, args))

    def generate_function_definitions(self, env, code):
        lenv = self.target.local_scope
        # Generate C code for header and body of function
3227
        code.mark_pos(self.pos)
3228 3229 3230 3231 3232 3233
        code.putln("")
        code.putln("/* Python wrapper */")
        preprocessor_guard = self.target.get_preprocessor_guard()
        if preprocessor_guard:
            code.putln(preprocessor_guard)

3234
        code.enter_cfunc_scope(lenv)
3235 3236
        code.return_from_error_cleanup_label = code.new_label()

Vitja Makarov's avatar
Vitja Makarov committed
3237
        with_pymethdef = (self.target.needs_assignment_synthesis(env, code) or
3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260
                          self.target.pymethdef_required)
        self.generate_function_header(code, with_pymethdef)
        self.generate_argument_declarations(lenv, code)
        tempvardecl_code = code.insertion_point()

        if self.return_type.is_pyobject:
            retval_init = ' = 0'
        else:
            retval_init = ''
        if not self.return_type.is_void:
            code.putln('%s%s;' % (
                self.return_type.declaration_code(Naming.retval_cname),
                retval_init))
        code.put_declare_refcount_context()
        code.put_setup_refcount_context('%s (wrapper)' % self.name)

        self.generate_argument_parsing_code(lenv, code)
        self.generate_argument_type_tests(code)
        self.generate_function_body(code)

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)

3261 3262 3263 3264
        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

3265 3266 3267 3268 3269 3270
        # ----- Error cleanup
        if code.error_label in code.labels_used:
            code.put_goto(code.return_label)
            code.put_label(code.error_label)
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
3271 3272 3273
            err_val = self.error_value()
            if err_val is not None:
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
        for entry in lenv.var_entries:
            if entry.is_arg and entry.type.is_pyobject:
                code.put_var_decref(entry)

        code.put_finish_refcount_context()
        if not self.return_type.is_void:
            code.putln("return %s;" % Naming.retval_cname)
        code.putln('}')
        code.exit_cfunc_scope()
        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

    def generate_function_header(self, code, with_pymethdef, proto_only=0):
        arg_code_list = []
        sig = self.signature

3293
        if sig.has_dummy_arg or self.self_in_stararg:
3294 3295 3296 3297
            arg_code = "PyObject *%s" % Naming.self_cname
            if not sig.has_dummy_arg:
                arg_code = 'CYTHON_UNUSED ' + arg_code
            arg_code_list.append(arg_code)
3298

William Stein's avatar
William Stein committed
3299 3300
        for arg in self.args:
            if not arg.is_generic:
3301
                if arg.is_self_arg or arg.is_type_arg:
William Stein's avatar
William Stein committed
3302 3303
                    arg_code_list.append("PyObject *%s" % arg.hdr_cname)
                else:
3304 3305 3306 3307
                    arg_code_list.append(
                        arg.hdr_type.declaration_code(arg.hdr_cname))
        entry = self.target.entry
        if not entry.is_special and sig.method_flags() == [TypeSlots.method_noargs]:
3308
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
3309
        if entry.scope.is_c_class_scope and entry.name == "__ipow__":
Lisandro Dalcin's avatar
Lisandro Dalcin committed
3310
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
William Stein's avatar
William Stein committed
3311 3312
        if sig.has_generic_args:
            arg_code_list.append(
3313 3314
                "PyObject *%s, PyObject *%s" % (
                    Naming.args_cname, Naming.kwds_cname))
William Stein's avatar
William Stein committed
3315
        arg_code = ", ".join(arg_code_list)
3316 3317 3318 3319

        # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__'
        mf = ""
        if (entry.name in ("__getbuffer__", "__releasebuffer__")
3320
                and entry.scope.is_c_class_scope):
3321
            mf = "CYTHON_UNUSED "
3322
            with_pymethdef = False
3323

3324
        dc = self.return_type.declaration_code(entry.func_cname)
3325
        header = "static %s%s(%s)" % (mf, dc, arg_code)
William Stein's avatar
William Stein committed
3326
        code.putln("%s; /*proto*/" % header)
3327

3328
        if proto_only:
3329
            if self.target.fused_py_func:
3330 3331 3332
                # If we are the specialized version of the cpdef, we still
                # want the prototype for the "fused cpdef", in case we're
                # checking to see if our method was overridden in Python
3333
                self.target.fused_py_func.generate_function_header(
3334
                    code, with_pymethdef, proto_only=True)
3335
            return
3336

3337 3338 3339 3340
        if (Options.docstrings and entry.doc and
                not self.target.fused_py_func and
                not entry.scope.is_property_scope and
                (not entry.is_special or entry.wrapperbase_cname)):
3341
            # h_code = code.globalstate['h_code']
3342
            docstr = entry.doc
3343

Stefan Behnel's avatar
Stefan Behnel committed
3344
            if docstr.is_unicode:
3345
                docstr = docstr.as_utf8_string()
3346

William Stein's avatar
William Stein committed
3347
            code.putln(
3348
                'static char %s[] = %s;' % (
3349
                    entry.doc_cname,
3350
                    docstr.as_c_string_literal()))
3351

3352
            if entry.is_special:
3353
                code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
3354
                code.putln(
3355
                    "struct wrapperbase %s;" % entry.wrapperbase_cname)
3356
                code.putln('#endif')
3357

3358
        if with_pymethdef or self.target.fused_py_func:
William Stein's avatar
William Stein committed
3359
            code.put(
3360
                "static PyMethodDef %s = " % entry.pymethdef_cname)
3361
            code.put_pymethoddef(self.target.entry, ";", allow_skip=False)
William Stein's avatar
William Stein committed
3362 3363 3364 3365
        code.putln("%s {" % header)

    def generate_argument_declarations(self, env, code):
        for arg in self.args:
3366
            if arg.is_generic:
3367 3368
                if arg.needs_conversion:
                    code.putln("PyObject *%s = 0;" % arg.hdr_cname)
3369
                else:
3370
                    code.put_var_declaration(arg.entry)
3371 3372 3373
        for entry in env.var_entries:
            if entry.is_arg:
                code.put_var_declaration(entry)
3374

3375
    def generate_argument_parsing_code(self, env, code):
Stefan Behnel's avatar
Stefan Behnel committed
3376 3377
        # Generate fast equivalent of PyArg_ParseTuple call for
        # generic arguments, if any, including args/kwargs
3378 3379
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
Stefan Behnel's avatar
Stefan Behnel committed
3380
        end_label = code.new_label("argument_unpacking_done")
3381

3382 3383 3384
        has_kwonly_args = self.num_kwonly_args > 0
        has_star_or_kw_args = self.star_arg is not None \
            or self.starstar_arg is not None or has_kwonly_args
3385

3386
        for arg in self.args:
3387
            if not arg.type.is_pyobject:
3388
                if not arg.type.create_from_py_utility_code(env):
3389
                    pass  # will fail later
3390

3391
        if not self.signature_has_generic_args():
3392 3393
            if has_star_or_kw_args:
                error(self.pos, "This method cannot have * or keyword arguments")
3394
            self.generate_argument_conversion_code(code)
3395

3396 3397
        elif not self.signature_has_nongeneric_args():
            # func(*args) or func(**kw) or func(*args, **kw)
3398
            self.generate_stararg_copy_code(code)
3399

3400
        else:
3401
            self.generate_tuple_and_keyword_parsing_code(self.args, end_label, code)
3402

3403 3404
        code.error_label = old_error_label
        if code.label_used(our_error_label):
3405 3406
            if not code.label_used(end_label):
                code.put_goto(end_label)
3407 3408 3409 3410 3411
            code.put_label(our_error_label)
            if has_star_or_kw_args:
                self.generate_arg_decref(self.star_arg, code)
                if self.starstar_arg:
                    if self.starstar_arg.entry.xdecref_cleanup:
3412
                        code.put_var_xdecref_clear(self.starstar_arg.entry)
3413
                    else:
3414
                        code.put_var_decref_clear(self.starstar_arg.entry)
3415
            code.put_add_traceback(self.target.entry.qualified_name)
3416
            code.put_finish_refcount_context()
3417
            code.putln("return %s;" % self.error_value())
3418
        if code.label_used(end_label):
3419 3420
            code.put_label(end_label)

William Stein's avatar
William Stein committed
3421 3422
    def generate_arg_xdecref(self, arg, code):
        if arg:
3423
            code.put_var_xdecref_clear(arg.entry)
3424

3425 3426
    def generate_arg_decref(self, arg, code):
        if arg:
3427
            code.put_var_decref_clear(arg.entry)
William Stein's avatar
William Stein committed
3428

3429 3430
    def generate_stararg_copy_code(self, code):
        if not self.star_arg:
3431 3432
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3433 3434 3435
            code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" %
                       Naming.args_cname)
            code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % (
3436
                self.name, Naming.args_cname, self.error_value()))
3437
            code.putln("}")
3438

3439
        if self.starstar_arg:
3440
            if self.star_arg or not self.starstar_arg.entry.cf_used:
3441 3442 3443 3444 3445 3446
                kwarg_check = "unlikely(%s)" % Naming.kwds_cname
            else:
                kwarg_check = "%s" % Naming.kwds_cname
        else:
            kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
                Naming.kwds_cname, Naming.kwds_cname)
3447 3448
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c"))
3449
        code.putln(
3450 3451
            "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
                kwarg_check, Naming.kwds_cname, self.name,
3452
                bool(self.starstar_arg), self.error_value()))
3453

3454 3455
        if self.starstar_arg and self.starstar_arg.entry.cf_used:
            if all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references):
3456 3457
                code.putln("if (%s) {" % kwarg_check)
                code.putln("%s = PyDict_Copy(%s); if (unlikely(!%s)) return %s;" % (
3458 3459 3460 3461
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    self.starstar_arg.entry.cname,
                    self.error_value()))
3462 3463
                code.put_gotref(self.starstar_arg.entry.cname)
                code.putln("} else {")
3464
                code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,))
3465 3466 3467
                code.putln("}")
                self.starstar_arg.entry.xdecref_cleanup = 1
            else:
3468
                code.put("%s = (%s) ? PyDict_Copy(%s) : PyDict_New(); " % (
3469 3470 3471
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    Naming.kwds_cname))
3472
                code.putln("if (unlikely(!%s)) return %s;" % (
3473
                    self.starstar_arg.entry.cname, self.error_value()))
3474 3475
                self.starstar_arg.entry.xdecref_cleanup = 0
                code.put_gotref(self.starstar_arg.entry.cname)
3476

3477
        if self.self_in_stararg and not self.target.is_staticmethod:
3478 3479
            # need to create a new tuple with 'self' inserted as first item
            code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % (
3480 3481 3482
                self.star_arg.entry.cname,
                Naming.args_cname,
                self.star_arg.entry.cname))
3483
            if self.starstar_arg and self.starstar_arg.entry.cf_used:
3484
                code.putln("{")
3485
                code.put_xdecref_clear(self.starstar_arg.entry.cname, py_object_type)
3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507
                code.putln("return %s;" % self.error_value())
                code.putln("}")
            else:
                code.putln("return %s;" % self.error_value())
            code.put_gotref(self.star_arg.entry.cname)
            code.put_incref(Naming.self_cname, py_object_type)
            code.put_giveref(Naming.self_cname)
            code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % (
                self.star_arg.entry.cname, Naming.self_cname))
            temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
            code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % (
                temp, temp, Naming.args_cname, temp))
            code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % (
                Naming.args_cname, temp))
            code.put_incref("item", py_object_type)
            code.put_giveref("item")
            code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % (
                self.star_arg.entry.cname, temp))
            code.putln("}")
            code.funcstate.release_temp(temp)
            self.star_arg.entry.xdecref_cleanup = 0
        elif self.star_arg:
3508 3509
            code.put_incref(Naming.args_cname, py_object_type)
            code.putln("%s = %s;" % (
3510 3511
                self.star_arg.entry.cname,
                Naming.args_cname))
3512 3513
            self.star_arg.entry.xdecref_cleanup = 0

3514
    def generate_tuple_and_keyword_parsing_code(self, args, success_label, code):
3515
        argtuple_error_label = code.new_label("argtuple_error")
3516

3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536
        positional_args = []
        required_kw_only_args = []
        optional_kw_only_args = []
        for arg in args:
            if arg.is_generic:
                if arg.default:
                    if not arg.is_self_arg and not arg.is_type_arg:
                        if arg.kw_only:
                            optional_kw_only_args.append(arg)
                        else:
                            positional_args.append(arg)
                elif arg.kw_only:
                    required_kw_only_args.append(arg)
                elif not arg.is_self_arg and not arg.is_type_arg:
                    positional_args.append(arg)

        # sort required kw-only args before optional ones to avoid special
        # cases in the unpacking code
        kw_only_args = required_kw_only_args + optional_kw_only_args

3537
        min_positional_args = self.num_required_args - self.num_required_kw_args
3538
        if len(args) > 0 and (args[0].is_self_arg or args[0].is_type_arg):
3539 3540
            min_positional_args -= 1
        max_positional_args = len(positional_args)
3541 3542
        has_fixed_positional_count = not self.star_arg and \
            min_positional_args == max_positional_args
3543
        has_kw_only_args = bool(kw_only_args)
3544

Stefan Behnel's avatar
Stefan Behnel committed
3545
        if self.num_required_kw_args:
3546 3547
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
Stefan Behnel's avatar
Stefan Behnel committed
3548

3549 3550 3551
        if self.starstar_arg or self.star_arg:
            self.generate_stararg_init_code(max_positional_args, code)

3552 3553
        code.putln('{')
        all_args = tuple(positional_args) + tuple(kw_only_args)
3554 3555
        code.putln("static PyObject **%s[] = {%s,0};" % (
            Naming.pykwdlist_cname,
3556 3557
            ','.join(['&%s' % code.intern_identifier(arg.name)
                      for arg in all_args])))
3558 3559 3560 3561 3562 3563 3564 3565 3566 3567

        # Before being converted and assigned to the target variables,
        # borrowed references to all unpacked argument values are
        # collected into a local PyObject* array called "values",
        # regardless if they were taken from default arguments,
        # positional arguments or keyword arguments.  Note that
        # C-typed default arguments are handled at conversion time,
        # so their array value is NULL in the end if no argument
        # was passed for them.
        self.generate_argument_values_setup_code(all_args, code)
3568

3569
        # --- optimised code when we receive keyword arguments
3570 3571 3572
        code.putln("if (%s(%s)) {" % (
            (self.num_required_kw_args > 0) and "likely" or "unlikely",
            Naming.kwds_cname))
3573 3574
        self.generate_keyword_unpacking_code(
            min_positional_args, max_positional_args,
3575 3576
            has_fixed_positional_count, has_kw_only_args,
            all_args, argtuple_error_label, code)
3577 3578

        # --- optimised code when we do not receive any keyword arguments
3579
        if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args:
3580 3581 3582 3583 3584 3585 3586
            # Python raises arg tuple related errors first, so we must
            # check the length here
            if min_positional_args == max_positional_args and not self.star_arg:
                compare = '!='
            else:
                compare = '<'
            code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % (
3587
                Naming.args_cname, compare, min_positional_args))
3588
            code.put_goto(argtuple_error_label)
3589

3590 3591 3592 3593
        if self.num_required_kw_args:
            # pure error case: keywords required but not passed
            if max_positional_args > min_positional_args and not self.star_arg:
                code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % (
3594
                    Naming.args_cname, max_positional_args))
3595 3596 3597 3598
                code.put_goto(argtuple_error_label)
            code.putln('} else {')
            for i, arg in enumerate(kw_only_args):
                if not arg.default:
3599
                    pystring_cname = code.intern_identifier(arg.name)
3600
                    # required keyword-only argument missing
3601
                    code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
3602 3603
                        self.name,
                        pystring_cname))
3604 3605
                    code.putln(code.error_goto(self.pos))
                    break
3606

3607
        else:
3608
            # optimised tuple unpacking code
3609
            code.putln('} else {')
3610 3611 3612 3613 3614
            if min_positional_args == max_positional_args:
                # parse the exact number of positional arguments from
                # the args tuple
                for i, arg in enumerate(positional_args):
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
3615
            else:
3616 3617 3618 3619 3620 3621 3622 3623
                # parse the positional arguments from the variable length
                # args tuple and reject illegal argument tuple sizes
                code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname)
                if self.star_arg:
                    code.putln('default:')
                reversed_args = list(enumerate(positional_args))[::-1]
                for i, arg in reversed_args:
                    if i >= min_positional_args-1:
3624
                        code.put('case %2d: ' % (i+1))
3625
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
3626
                    code.putln('CYTHON_FALLTHROUGH')
3627 3628 3629 3630 3631 3632 3633 3634 3635 3636
                if min_positional_args == 0:
                    code.put('case  0: ')
                code.putln('break;')
                if self.star_arg:
                    if min_positional_args:
                        for i in range(min_positional_args-1, -1, -1):
                            code.putln('case %2d:' % i)
                        code.put_goto(argtuple_error_label)
                else:
                    code.put('default: ')
3637
                    code.put_goto(argtuple_error_label)
3638 3639
                code.putln('}')

3640
        code.putln('}') # end of the conditional unpacking blocks
3641

3642 3643 3644
        # Convert arg values to their final type and assign them.
        # Also inject non-Python default arguments, which do cannot
        # live in the values[] array.
3645 3646
        for i, arg in enumerate(all_args):
            self.generate_arg_assignment(arg, "values[%d]" % i, code)
3647

3648
        code.putln('}') # end of the whole argument unpacking block
3649 3650 3651 3652

        if code.label_used(argtuple_error_label):
            code.put_goto(success_label)
            code.put_label(argtuple_error_label)
3653 3654
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3655
            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
3656 3657 3658
                self.name, has_fixed_positional_count,
                min_positional_args, max_positional_args,
                Naming.args_cname))
3659 3660
            code.putln(code.error_goto(self.pos))

3661 3662 3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682
    def generate_arg_assignment(self, arg, item, code):
        if arg.type.is_pyobject:
            # Python default arguments were already stored in 'item' at the very beginning
            if arg.is_generic:
                item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
            entry = arg.entry
            code.putln("%s = %s;" % (entry.cname, item))
        else:
            func = arg.type.from_py_function
            if func:
                if arg.default:
                    # C-typed default arguments must be handled here
                    code.putln('if (%s) {' % item)
                rhs = "%s(%s)" % (func, item)
                if arg.type.is_enum:
                    rhs = arg.type.cast_code(rhs)
                code.putln("%s = %s; %s" % (
                    arg.entry.cname,
                    rhs,
                    code.error_goto_if(arg.type.error_condition(arg.entry.cname), arg.pos)))
                if arg.default:
                    code.putln('} else {')
3683 3684 3685
                    code.putln("%s = %s;" % (
                        arg.entry.cname,
                        arg.calculate_default_value_code(code)))
3686 3687 3688 3689 3690 3691
                    if arg.type.is_memoryviewslice:
                        code.put_incref_memoryviewslice(arg.entry.cname,
                                                        have_gil=True)
                    code.putln('}')
            else:
                error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type)
3692

3693
    def generate_stararg_init_code(self, max_positional_args, code):
3694
        if self.starstar_arg:
3695
            self.starstar_arg.entry.xdecref_cleanup = 0
Stefan Behnel's avatar
Stefan Behnel committed
3696
            code.putln('%s = PyDict_New(); if (unlikely(!%s)) return %s;' % (
3697 3698 3699
                self.starstar_arg.entry.cname,
                self.starstar_arg.entry.cname,
                self.error_value()))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3700
            code.put_gotref(self.starstar_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3701 3702 3703
        if self.star_arg:
            self.star_arg.entry.xdecref_cleanup = 0
            code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % (
3704 3705
                Naming.args_cname,
                max_positional_args))
3706
            code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % (
3707 3708
                self.star_arg.entry.cname, Naming.args_cname,
                max_positional_args, Naming.args_cname))
3709
            code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3710
            if self.starstar_arg:
3711
                code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
3712 3713 3714 3715
            code.put_finish_refcount_context()
            code.putln('return %s;' % self.error_value())
            code.putln('}')
            code.put_gotref(self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3716 3717 3718 3719
            code.putln('} else {')
            code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple))
            code.put_incref(Naming.empty_tuple, py_object_type)
            code.putln('}')
3720

3721
    def generate_argument_values_setup_code(self, args, code):
3722
        max_args = len(args)
Stefan Behnel's avatar
Stefan Behnel committed
3723 3724
        # the 'values' array collects borrowed references to arguments
        # before doing any type coercion etc.
3725
        code.putln("PyObject* values[%d] = {%s};" % (
Stefan Behnel's avatar
Stefan Behnel committed
3726
            max_args, ','.join('0'*max_args)))
3727

3728
        if self.target.defaults_struct:
3729
            code.putln('%s *%s = __Pyx_CyFunction_Defaults(%s, %s);' % (
3730 3731
                self.target.defaults_struct, Naming.dynamic_args_cname,
                self.target.defaults_struct, Naming.self_cname))
3732

3733 3734
        # assign borrowed Python default values to the values array,
        # so that they can be overwritten by received arguments below
3735
        for i, arg in enumerate(args):
3736 3737 3738 3739
            if arg.default and arg.type.is_pyobject:
                default_value = arg.calculate_default_value_code(code)
                code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value)))

3740 3741 3742 3743
    def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args,
                                        has_fixed_positional_count, has_kw_only_args,
                                        all_args, argtuple_error_label, code):
        code.putln('Py_ssize_t kw_args;')
3744
        code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname)
3745
        # copy the values from the args tuple and check that it's not too long
3746
        code.putln('switch (pos_args) {')
Stefan Behnel's avatar
Stefan Behnel committed
3747 3748
        if self.star_arg:
            code.putln('default:')
3749
        for i in range(max_positional_args-1, -1, -1):
3750
            code.put('case %2d: ' % (i+1))
3751
            code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (
3752
                i, Naming.args_cname, i))
3753
            code.putln('CYTHON_FALLTHROUGH')
3754
        code.putln('case  0: break;')
3755
        if not self.star_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3756
            code.put('default: ') # more arguments than allowed
3757
            code.put_goto(argtuple_error_label)
3758 3759
        code.putln('}')

3760 3761 3762 3763 3764 3765 3766 3767 3768
        # The code above is very often (but not always) the same as
        # the optimised non-kwargs tuple unpacking code, so we keep
        # the code block above at the very top, before the following
        # 'external' PyDict_Size() call, to make it easy for the C
        # compiler to merge the two separate tuple unpacking
        # implementations into one when they turn out to be identical.

        # If we received kwargs, fill up the positional/required
        # arguments with values from the kw dict
3769
        code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname)
3770
        if self.num_required_args or max_positional_args > 0:
Stefan Behnel's avatar
Stefan Behnel committed
3771 3772 3773 3774
            last_required_arg = -1
            for i, arg in enumerate(all_args):
                if not arg.default:
                    last_required_arg = i
3775 3776
            if last_required_arg < max_positional_args:
                last_required_arg = max_positional_args-1
Stefan Behnel's avatar
Stefan Behnel committed
3777
            if max_positional_args > 0:
3778
                code.putln('switch (pos_args) {')
3779
            for i, arg in enumerate(all_args[:last_required_arg+1]):
Stefan Behnel's avatar
Stefan Behnel committed
3780
                if max_positional_args > 0 and i <= max_positional_args:
3781 3782 3783 3784
                    if self.star_arg and i == max_positional_args:
                        code.putln('default:')
                    else:
                        code.putln('case %2d:' % i)
3785
                pystring_cname = code.intern_identifier(arg.name)
3786
                if arg.default:
3787
                    if arg.kw_only:
3788
                        # optional kw-only args are handled separately below
3789
                        continue
3790
                    code.putln('if (kw_args > 0) {')
3791
                    # don't overwrite default argument
3792
                    code.putln('PyObject* value = PyDict_GetItem(%s, %s);' % (
3793
                        Naming.kwds_cname, pystring_cname))
3794
                    code.putln('if (value) { values[%d] = value; kw_args--; }' % i)
3795 3796
                    code.putln('}')
                else:
3797
                    code.putln('if (likely((values[%d] = PyDict_GetItem(%s, %s)) != 0)) kw_args--;' % (
3798
                        i, Naming.kwds_cname, pystring_cname))
3799 3800 3801 3802 3803 3804 3805 3806 3807 3808
                    if i < min_positional_args:
                        if i == 0:
                            # special case: we know arg 0 is missing
                            code.put('else ')
                            code.put_goto(argtuple_error_label)
                        else:
                            # print the correct number of values (args or
                            # kwargs) that were passed into positional
                            # arguments up to this point
                            code.putln('else {')
3809 3810
                            code.globalstate.use_utility_code(
                                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3811
                            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
3812 3813
                                self.name, has_fixed_positional_count,
                                min_positional_args, max_positional_args, i))
3814 3815 3816
                            code.putln(code.error_goto(self.pos))
                            code.putln('}')
                    elif arg.kw_only:
3817
                        code.putln('else {')
3818 3819
                        code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
                            self.name, pystring_cname))
3820 3821
                        code.putln(code.error_goto(self.pos))
                        code.putln('}')
3822 3823
                if max_positional_args > 0 and i < last_required_arg:
                    code.putln('CYTHON_FALLTHROUGH')
Stefan Behnel's avatar
Stefan Behnel committed
3824 3825
            if max_positional_args > 0:
                code.putln('}')
3826

3827
        if has_kw_only_args:
3828
            # unpack optional keyword-only arguments separately because
3829
            # checking for interned strings in a dict is faster than iterating
3830
            self.generate_optional_kwonly_args_unpacking_code(all_args, code)
3831

3832
        code.putln('if (unlikely(kw_args > 0)) {')
3833 3834
        # non-positional/-required kw args left in dict: default args,
        # kw-only args, **kwargs or error
Stefan Behnel's avatar
Stefan Behnel committed
3835 3836 3837 3838 3839
        #
        # This is sort of a catch-all: except for checking required
        # arguments, this will always do the right thing for unpacking
        # keyword arguments, so that we can concentrate on optimising
        # common cases above.
3840 3841 3842
        if max_positional_args == 0:
            pos_arg_count = "0"
        elif self.star_arg:
3843
            code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % (
3844
                max_positional_args, max_positional_args))
Stefan Behnel's avatar
Stefan Behnel committed
3845 3846
            pos_arg_count = "used_pos_args"
        else:
3847
            pos_arg_count = "pos_args"
3848 3849
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
3850 3851 3852 3853 3854 3855 3856
        code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % (
            Naming.kwds_cname,
            Naming.pykwdlist_cname,
            self.starstar_arg and self.starstar_arg.entry.cname or '0',
            pos_arg_count,
            self.name,
            code.error_goto(self.pos)))
3857
        code.putln('}')
3858

3859 3860 3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889
    def generate_optional_kwonly_args_unpacking_code(self, all_args, code):
        optional_args = []
        first_optional_arg = -1
        for i, arg in enumerate(all_args):
            if not arg.kw_only or not arg.default:
                continue
            if not optional_args:
                first_optional_arg = i
            optional_args.append(arg.name)
        if optional_args:
            if len(optional_args) > 1:
                # if we receive more than the named kwargs, we either have **kwargs
                # (in which case we must iterate anyway) or it's an error (which we
                # also handle during iteration) => skip this part if there are more
                code.putln('if (kw_args > 0 && %s(kw_args <= %d)) {' % (
                    not self.starstar_arg and 'likely' or '',
                    len(optional_args)))
                code.putln('Py_ssize_t index;')
                # not unrolling the loop here reduces the C code overhead
                code.putln('for (index = %d; index < %d && kw_args > 0; index++) {' % (
                    first_optional_arg, first_optional_arg + len(optional_args)))
            else:
                code.putln('if (kw_args == 1) {')
                code.putln('const Py_ssize_t index = %d;' % first_optional_arg)
            code.putln('PyObject* value = PyDict_GetItem(%s, *%s[index]);' % (
                Naming.kwds_cname, Naming.pykwdlist_cname))
            code.putln('if (value) { values[index] = value; kw_args--; }')
            if len(optional_args) > 1:
                code.putln('}')
            code.putln('}')

William Stein's avatar
William Stein committed
3890
    def generate_argument_conversion_code(self, code):
3891 3892 3893
        # Generate code to convert arguments from signature type to
        # declared type, if needed.  Also copies signature arguments
        # into closure fields.
William Stein's avatar
William Stein committed
3894 3895 3896 3897 3898 3899 3900 3901 3902
        for arg in self.args:
            if arg.needs_conversion:
                self.generate_arg_conversion(arg, code)

    def generate_arg_conversion(self, arg, code):
        # Generate conversion code for one argument.
        old_type = arg.hdr_type
        new_type = arg.type
        if old_type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
3903 3904 3905 3906
            if arg.default:
                code.putln("if (%s) {" % arg.hdr_cname)
            else:
                code.putln("assert(%s); {" % arg.hdr_cname)
William Stein's avatar
William Stein committed
3907
            self.generate_arg_conversion_from_pyobject(arg, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
3908
            code.putln("}")
William Stein's avatar
William Stein committed
3909 3910 3911 3912
        elif new_type.is_pyobject:
            self.generate_arg_conversion_to_pyobject(arg, code)
        else:
            if new_type.assignable_from(old_type):
3913
                code.putln("%s = %s;" % (arg.entry.cname, arg.hdr_cname))
William Stein's avatar
William Stein committed
3914
            else:
3915
                error(arg.pos, "Cannot convert 1 argument from '%s' to '%s'" % (old_type, new_type))
3916

William Stein's avatar
William Stein committed
3917 3918 3919
    def generate_arg_conversion_from_pyobject(self, arg, code):
        new_type = arg.type
        func = new_type.from_py_function
3920
        # copied from CoerceFromPyTypeNode
William Stein's avatar
William Stein committed
3921
        if func:
3922 3923 3924 3925 3926
            lhs = arg.entry.cname
            rhs = "%s(%s)" % (func, arg.hdr_cname)
            if new_type.is_enum:
                rhs = PyrexTypes.typecast(new_type, PyrexTypes.c_long_type, rhs)
            code.putln("%s = %s; %s" % (
3927
                lhs,
3928
                rhs,
3929
                code.error_goto_if(new_type.error_condition(arg.entry.cname), arg.pos)))
William Stein's avatar
William Stein committed
3930
        else:
3931
            error(arg.pos, "Cannot convert Python object argument to type '%s'" % new_type)
3932

William Stein's avatar
William Stein committed
3933 3934 3935 3936
    def generate_arg_conversion_to_pyobject(self, arg, code):
        old_type = arg.hdr_type
        func = old_type.to_py_function
        if func:
Robert Bradshaw's avatar
Robert Bradshaw committed
3937
            code.putln("%s = %s(%s); %s" % (
William Stein's avatar
William Stein committed
3938 3939 3940
                arg.entry.cname,
                func,
                arg.hdr_cname,
Robert Bradshaw's avatar
Robert Bradshaw committed
3941
                code.error_goto_if_null(arg.entry.cname, arg.pos)))
3942
            code.put_var_gotref(arg.entry)
William Stein's avatar
William Stein committed
3943
        else:
3944
            error(arg.pos, "Cannot convert argument of type '%s' to Python object" % old_type)
William Stein's avatar
William Stein committed
3945 3946 3947 3948 3949 3950 3951 3952

    def generate_argument_type_tests(self, code):
        # Generate type tests for args whose signature
        # type is PyObject * and whose declared type is
        # a subtype thereof.
        for arg in self.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
3953 3954 3955
            elif not arg.accept_none and (arg.type.is_pyobject or
                                          arg.type.is_buffer or
                                          arg.type.is_memoryviewslice):
3956 3957
                self.generate_arg_none_check(arg, code)

William Stein's avatar
William Stein committed
3958
    def error_value(self):
3959
        return self.signature.error_value
3960

3961 3962

class GeneratorDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
3963
    # Generator function node that creates a new generator instance when called.
3964
    #
Stefan Behnel's avatar
Stefan Behnel committed
3965
    # gbody          GeneratorBodyDefNode   the function implementing the generator
3966 3967 3968
    #

    is_generator = True
3969
    is_coroutine = False
3970 3971
    needs_closure = True

Stefan Behnel's avatar
Stefan Behnel committed
3972
    child_attrs = DefNode.child_attrs + ["gbody"]
3973

3974
    def __init__(self, pos, **kwargs):
3975
        # XXX: don't actually needs a body
3976 3977
        kwargs['body'] = StatListNode(pos, stats=[], is_terminator=True)
        super(GeneratorDefNode, self).__init__(pos, **kwargs)
3978 3979 3980 3981 3982 3983 3984 3985

    def analyse_declarations(self, env):
        super(GeneratorDefNode, self).analyse_declarations(env)
        self.gbody.local_scope = self.local_scope
        self.gbody.analyse_declarations(env)

    def generate_function_body(self, env, code):
        body_cname = self.gbody.entry.func_cname
3986 3987
        name = code.intern_identifier(self.name)
        qualname = code.intern_identifier(self.qualname)
3988
        module_name = code.intern_identifier(self.module_name)
3989

3990
        code.putln('{')
3991
        code.putln('__pyx_CoroutineObject *gen = __Pyx_%s_New('
3992
                   '(__pyx_coroutine_body_t) %s, (PyObject *) %s, %s, %s, %s); %s' % (
3993
                       'Coroutine' if self.is_coroutine else 'Generator',
3994
                       body_cname, Naming.cur_scope_cname, name, qualname, module_name,
3995 3996
                       code.error_goto_if_null('gen', self.pos)))
        code.put_decref(Naming.cur_scope_cname, py_object_type)
3997
        if self.requires_classobj:
3998
            classobj_cname = 'gen->classobj'
3999 4000 4001 4002
            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                classobj_cname, Naming.self_cname))
            code.put_incref(classobj_cname, py_object_type)
            code.put_giveref(classobj_cname)
4003
        code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4004
        code.putln('return (PyObject *) gen;')
4005
        code.putln('}')
4006 4007

    def generate_function_definitions(self, env, code):
4008 4009
        env.use_utility_code(UtilityCode.load_cached(
            'Coroutine' if self.is_coroutine else 'Generator', "Coroutine.c"))
4010

4011 4012 4013 4014 4015
        self.gbody.generate_function_header(code, proto=True)
        super(GeneratorDefNode, self).generate_function_definitions(env, code)
        self.gbody.generate_function_definitions(env, code)


4016 4017 4018 4019
class AsyncDefNode(GeneratorDefNode):
    is_coroutine = True


4020
class GeneratorBodyDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
4021
    # Main code body of a generator implemented as a DefNode.
4022 4023 4024
    #

    is_generator_body = True
4025
    is_inlined = False
4026
    inlined_comprehension_type = None  # container type for inlined comprehensions
4027

4028
    def __init__(self, pos=None, name=None, body=None):
4029 4030 4031
        super(GeneratorBodyDefNode, self).__init__(
            pos=pos, body=body, name=name, doc=None,
            args=[], star_arg=None, starstar_arg=None)
4032

4033 4034
    def declare_generator_body(self, env):
        prefix = env.next_id(env.scope_prefix)
Vitja Makarov's avatar
Vitja Makarov committed
4035
        name = env.next_id('generator')
4036 4037 4038 4039
        cname = Naming.genbody_prefix + prefix + name
        entry = env.declare_var(None, py_object_type, self.pos,
                                cname=cname, visibility='private')
        entry.func_cname = cname
4040 4041 4042 4043 4044 4045
        entry.qualified_name = EncodedString(self.name)
        self.entry = entry

    def analyse_declarations(self, env):
        self.analyse_argument_types(env)
        self.declare_generator_body(env)
4046 4047

    def generate_function_header(self, code, proto=False):
4048
        header = "static PyObject *%s(__pyx_CoroutineObject *%s, PyObject *%s)" % (
4049
            self.entry.func_cname,
4050
            Naming.generator_cname,
4051 4052 4053 4054
            Naming.sent_value_cname)
        if proto:
            code.putln('%s; /* proto */' % header)
        else:
Stefan Behnel's avatar
Stefan Behnel committed
4055
            code.putln('%s /* generator body */\n{' % header)
4056 4057 4058 4059 4060 4061 4062 4063

    def generate_function_definitions(self, env, code):
        lenv = self.local_scope

        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)

        # Generate C code for header and body of function
4064
        code.enter_cfunc_scope(lenv)
4065 4066 4067 4068 4069 4070 4071 4072
        code.return_from_error_cleanup_label = code.new_label()

        # ----- Top-level constants used by this function
        code.mark_pos(self.pos)
        self.generate_cached_builtins_decls(lenv, code)
        # ----- Function header
        code.putln("")
        self.generate_function_header(code)
4073
        closure_init_code = code.insertion_point()
4074 4075 4076 4077 4078
        # ----- Local variables
        code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
        tempvardecl_code = code.insertion_point()
        code.put_declare_refcount_context()
        code.put_setup_refcount_context(self.entry.name)
4079 4080 4081 4082
        profile = code.globalstate.directives['profile']
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
            code.put_trace_declarations()
4083 4084 4085 4086 4087 4088 4089 4090 4091 4092

        # ----- Resume switch point.
        code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
        resume_code = code.insertion_point()
        first_run_label = code.new_label('first_run')
        code.use_label(first_run_label)
        code.put_label(first_run_label)
        code.putln('%s' %
                   (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))

4093 4094 4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109
        # ----- prepare target container for inlined comprehension
        if self.is_inlined and self.inlined_comprehension_type is not None:
            target_type = self.inlined_comprehension_type
            if target_type is Builtin.list_type:
                comp_init = 'PyList_New(0)'
            elif target_type is Builtin.set_type:
                comp_init = 'PySet_New(NULL)'
            elif target_type is Builtin.dict_type:
                comp_init = 'PyDict_New()'
            else:
                raise InternalError(
                    "invalid type of inlined comprehension: %s" % target_type)
            code.putln("%s = %s; %s" % (
                Naming.retval_cname, comp_init,
                code.error_goto_if_null(Naming.retval_cname, self.pos)))
            code.put_gotref(Naming.retval_cname)

4110 4111
        # ----- Function body
        self.generate_function_body(env, code)
4112 4113 4114 4115 4116 4117
        # ----- Closure initialization
        if lenv.scope_class.type.scope.entries:
            closure_init_code.putln('%s = %s;' % (
                lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
                lenv.scope_class.type.cast_code('%s->closure' %
                                                Naming.generator_cname)))
4118
            # FIXME: this silences a potential "unused" warning => try to avoid unused closures in more cases
4119
            code.putln("CYTHON_MAYBE_UNUSED_VAR(%s);" % Naming.cur_scope_cname)
4120 4121 4122 4123 4124

        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

Stefan Behnel's avatar
Stefan Behnel committed
4125 4126
        # on normal generator termination, we do not take the exception propagation
        # path: no traceback info is required and not creating it is much faster
4127
        if not self.is_inlined and not self.body.is_terminator:
4128
            code.putln('PyErr_SetNone(PyExc_StopIteration);')
4129 4130
        # ----- Error cleanup
        if code.error_label in code.labels_used:
4131 4132
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
4133
            code.put_label(code.error_label)
4134 4135
            if self.is_inlined and self.inlined_comprehension_type is not None:
                code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4136 4137
            if Future.generator_stop in env.global_scope().context.future_directives:
                # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
4138
                code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Coroutine.c"))
4139
                code.putln("if (unlikely(PyErr_ExceptionMatches(PyExc_StopIteration))) "
4140
                           "__Pyx_Generator_Replace_StopIteration();")
4141 4142
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
4143
            code.put_add_traceback(self.entry.qualified_name)
4144 4145 4146

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
4147 4148 4149 4150
        if self.is_inlined:
            code.put_xgiveref(Naming.retval_cname)
        else:
            code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4151
        code.putln('%s->resume_label = -1;' % Naming.generator_cname)
4152
        # clean up as early as possible to help breaking any reference cycles
4153
        code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname)
4154 4155 4156
        if profile or linetrace:
            code.put_trace_return(Naming.retval_cname,
                                  nogil=not code.funcstate.gil_owned)
4157
        code.put_finish_refcount_context()
4158
        code.putln("return %s;" % Naming.retval_cname)
4159 4160 4161 4162 4163
        code.putln("}")

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)
        # ----- Generator resume code
4164 4165 4166
        if profile or linetrace:
            resume_code.put_trace_call(self.entry.qualified_name, self.pos,
                                       nogil=not code.funcstate.gil_owned)
4167
        resume_code.putln("switch (%s->resume_label) {" % (
Stefan Behnel's avatar
Stefan Behnel committed
4168
                       Naming.generator_cname))
4169

4170
        resume_code.putln("case 0: goto %s;" % first_run_label)
4171

4172 4173
        for i, label in code.yield_labels:
            resume_code.putln("case %d: goto %s;" % (i, label))
Stefan Behnel's avatar
Stefan Behnel committed
4174
        resume_code.putln("default: /* CPython raises the right error here */")
4175 4176 4177
        if profile or linetrace:
            resume_code.put_trace_return("Py_None",
                                         nogil=not code.funcstate.gil_owned)
4178
        resume_code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4179 4180
        resume_code.putln("return NULL;")
        resume_code.putln("}")
4181 4182 4183 4184

        code.exit_cfunc_scope()


4185 4186
class OverrideCheckNode(StatNode):
    # A Node for dispatching to the def method if it
4187
    # is overriden.
4188 4189 4190 4191 4192 4193
    #
    #  py_func
    #
    #  args
    #  func_temp
    #  body
4194

Robert Bradshaw's avatar
Robert Bradshaw committed
4195
    child_attrs = ['body']
4196

4197
    body = None
Robert Bradshaw's avatar
Robert Bradshaw committed
4198

4199 4200
    def analyse_expressions(self, env):
        self.args = env.arg_entries
4201 4202 4203 4204
        if self.py_func.is_module_scope:
            first_arg = 0
        else:
            first_arg = 1
4205
        from . import ExprNodes
4206
        self.func_node = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
Stefan Behnel's avatar
Stefan Behnel committed
4207 4208
        call_node = ExprNodes.SimpleCallNode(
            self.pos, function=self.func_node,
4209 4210
            args=[ExprNodes.NameNode(self.pos, name=arg.name)
                  for arg in self.args[first_arg:]])
4211 4212
        if env.return_type.is_void or env.return_type.is_returncode:
            self.body = StatListNode(self.pos, stats=[
4213 4214
                ExprStatNode(self.pos, expr=call_node),
                ReturnStatNode(self.pos, value=None)])
4215 4216
        else:
            self.body = ReturnStatNode(self.pos, value=call_node)
4217 4218
        self.body = self.body.analyse_expressions(env)
        return self
4219

4220
    def generate_execution_code(self, code):
4221
        interned_attr_cname = code.intern_identifier(self.py_func.entry.name)
4222
        # Check to see if we are an extension type
4223 4224 4225 4226
        if self.py_func.is_module_scope:
            self_arg = "((PyObject *)%s)" % Naming.module_cname
        else:
            self_arg = "((PyObject *)%s)" % self.args[0].cname
4227
        code.putln("/* Check if called by wrapper */")
4228
        code.putln("if (unlikely(%s)) ;" % Naming.skip_dispatch_cname)
Stefan Behnel's avatar
Stefan Behnel committed
4229
        code.putln("/* Check if overridden in Python */")
4230 4231 4232
        if self.py_func.is_module_scope:
            code.putln("else {")
        else:
4233
            code.putln("else if (unlikely(Py_TYPE(%s)->tp_dictoffset != 0)) {" % self_arg)
4234 4235
        func_node_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.func_node.set_cname(func_node_temp)
4236
        # need to get attribute manually--scope would return cdef method
4237 4238
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("PyObjectGetAttrStr", "ObjectHandling.c"))
4239
        err = code.error_goto_if_null(func_node_temp, self.pos)
4240
        code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % (
4241 4242 4243
            func_node_temp, self_arg, interned_attr_cname, err))
        code.put_gotref(func_node_temp)
        is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp
4244
        is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)%s)" % (
4245
            func_node_temp, self.py_func.entry.func_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
4246
        code.putln("if (!%s || %s) {" % (is_builtin_function_or_method, is_overridden))
4247 4248
        self.body.generate_execution_code(code)
        code.putln("}")
4249 4250
        code.put_decref_clear(func_node_temp, PyrexTypes.py_object_type)
        code.funcstate.release_temp(func_node_temp)
Robert Bradshaw's avatar
Robert Bradshaw committed
4251
        code.putln("}")
4252

4253

Robert Bradshaw's avatar
Robert Bradshaw committed
4254 4255
class ClassDefNode(StatNode, BlockNode):
    pass
4256

4257

Robert Bradshaw's avatar
Robert Bradshaw committed
4258
class PyClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4259 4260
    #  A Python class definition.
    #
Stefan Behnel's avatar
Stefan Behnel committed
4261
    #  name     EncodedString   Name of the class
William Stein's avatar
William Stein committed
4262 4263 4264 4265
    #  doc      string or None
    #  body     StatNode        Attribute definition code
    #  entry    Symtab.Entry
    #  scope    PyClassScope
4266
    #  decorators    [DecoratorNode]        list of decorators or None
William Stein's avatar
William Stein committed
4267 4268 4269
    #
    #  The following subnodes are constructed internally:
    #
4270
    #  dict     DictNode   Class dictionary or Py3 namespace
William Stein's avatar
William Stein committed
4271 4272
    #  classobj ClassNode  Class object
    #  target   NameNode   Variable to assign class object to
4273

4274
    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result",
4275
                   "target", "class_cell", "decorators"]
4276
    decorators = None
4277
    class_result = None
4278 4279 4280
    is_py3_style_class = False  # Python3 style class (kwargs)
    metaclass = None
    mkw = None
4281

4282
    def __init__(self, pos, name, bases, doc, body, decorators=None,
4283
                 keyword_args=None, force_py3_semantics=False):
William Stein's avatar
William Stein committed
4284 4285 4286 4287
        StatNode.__init__(self, pos)
        self.name = name
        self.doc = doc
        self.body = body
4288
        self.decorators = decorators
4289
        self.bases = bases
4290
        from . import ExprNodes
4291
        if self.doc and Options.docstrings:
4292
            doc = embed_position(self.pos, self.doc)
4293
            doc_node = ExprNodes.StringNode(pos, value=doc)
William Stein's avatar
William Stein committed
4294 4295
        else:
            doc_node = None
4296 4297

        allow_py2_metaclass = not force_py3_semantics
4298
        if keyword_args:
4299 4300
            allow_py2_metaclass = False
            self.is_py3_style_class = True
4301 4302 4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314
            if keyword_args.is_dict_literal:
                if keyword_args.key_value_pairs:
                    for i, item in list(enumerate(keyword_args.key_value_pairs))[::-1]:
                        if item.key.value == 'metaclass':
                            if self.metaclass is not None:
                                error(item.pos, "keyword argument 'metaclass' passed multiple times")
                            # special case: we already know the metaclass,
                            # so we don't need to do the "build kwargs,
                            # find metaclass" dance at runtime
                            self.metaclass = item.value
                            del keyword_args.key_value_pairs[i]
                    self.mkw = keyword_args
                else:
                    assert self.metaclass is not None
4315
            else:
4316
                # MergedDictNode
4317
                self.mkw = ExprNodes.ProxyNode(keyword_args)
4318 4319

        if force_py3_semantics or self.bases or self.mkw or self.metaclass:
4320
            if self.metaclass is None:
4321
                if keyword_args and not keyword_args.is_dict_literal:
4322 4323 4324 4325
                    # **kwargs may contain 'metaclass' arg
                    mkdict = self.mkw
                else:
                    mkdict = None
4326 4327 4328 4329 4330 4331 4332
                if (not mkdict and
                        self.bases.is_sequence_constructor and
                        not self.bases.args):
                    pass  # no base classes => no inherited metaclass
                else:
                    self.metaclass = ExprNodes.PyClassMetaclassNode(
                        pos, mkw=mkdict, bases=self.bases)
4333 4334 4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345
                needs_metaclass_calculation = False
            else:
                needs_metaclass_calculation = True

            self.dict = ExprNodes.PyClassNamespaceNode(
                pos, name=name, doc=doc_node,
                metaclass=self.metaclass, bases=self.bases, mkw=self.mkw)
            self.classobj = ExprNodes.Py3ClassNode(
                pos, name=name,
                bases=self.bases, dict=self.dict, doc=doc_node,
                metaclass=self.metaclass, mkw=self.mkw,
                calculate_metaclass=needs_metaclass_calculation,
                allow_py2_metaclass=allow_py2_metaclass)
4346
        else:
4347 4348 4349 4350 4351 4352 4353
            # no bases, no metaclass => old style class creation
            self.dict = ExprNodes.DictNode(pos, key_value_pairs=[])
            self.classobj = ExprNodes.ClassNode(
                pos, name=name,
                bases=bases, dict=self.dict, doc=doc_node)

        self.target = ExprNodes.NameNode(pos, name=name)
4354
        self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
4355

4356 4357
    def as_cclass(self):
        """
4358
        Return this node as if it were declared as an extension class
4359
        """
4360
        if self.is_py3_style_class:
4361 4362
            error(self.classobj.pos, "Python3 style class could not be represented as C class")
            return
4363 4364 4365 4366 4367 4368 4369
        bases = self.classobj.bases.args
        if len(bases) == 0:
            base_class_name = None
            base_class_module = None
        elif len(bases) == 1:
            base = bases[0]
            path = []
4370
            from .ExprNodes import AttributeNode, NameNode
4371
            while isinstance(base, AttributeNode):
4372 4373
                path.insert(0, base.attribute)
                base = base.obj
4374
            if isinstance(base, NameNode):
4375 4376 4377 4378 4379 4380 4381 4382 4383 4384 4385
                path.insert(0, base.name)
                base_class_name = path[-1]
                if len(path) > 1:
                    base_class_module = u'.'.join(path[:-1])
                else:
                    base_class_module = None
            else:
                error(self.classobj.bases.args.pos, "Invalid base class")
        else:
            error(self.classobj.bases.args.pos, "C class may only have one base class")
            return None
4386 4387

        return CClassDefNode(self.pos,
4388 4389 4390 4391 4392 4393 4394 4395 4396
                             visibility='private',
                             module_name=None,
                             class_name=self.name,
                             base_class_module=base_class_module,
                             base_class_name=base_class_name,
                             decorators=self.decorators,
                             body=self.body,
                             in_pxd=False,
                             doc=self.doc)
4397

4398 4399
    def create_scope(self, env):
        genv = env
4400 4401
        while genv.is_py_class_scope or genv.is_c_class_scope:
            genv = genv.outer_scope
4402
        cenv = self.scope = PyClassScope(name=self.name, outer_scope=genv)
4403
        return cenv
4404

William Stein's avatar
William Stein committed
4405
    def analyse_declarations(self, env):
4406 4407
        class_result = self.classobj
        if self.decorators:
4408
            from .ExprNodes import SimpleCallNode
4409 4410 4411
            for decorator in self.decorators[::-1]:
                class_result = SimpleCallNode(
                    decorator.pos,
4412 4413
                    function=decorator.decorator,
                    args=[class_result])
4414
            self.decorators = None
4415 4416
        self.class_result = class_result
        self.class_result.analyse_declarations(env)
William Stein's avatar
William Stein committed
4417
        self.target.analyse_target_declaration(env)
4418
        cenv = self.create_scope(env)
4419
        cenv.directives = env.directives
4420 4421
        cenv.class_obj_cname = self.target.entry.cname
        self.body.analyse_declarations(cenv)
4422

William Stein's avatar
William Stein committed
4423
    def analyse_expressions(self, env):
4424
        if self.bases:
4425
            self.bases = self.bases.analyse_expressions(env)
4426
        if self.metaclass:
4427
            self.metaclass = self.metaclass.analyse_expressions(env)
4428
        if self.mkw:
4429 4430 4431
            self.mkw = self.mkw.analyse_expressions(env)
        self.dict = self.dict.analyse_expressions(env)
        self.class_result = self.class_result.analyse_expressions(env)
4432
        cenv = self.scope
4433
        self.body = self.body.analyse_expressions(cenv)
4434
        self.target.analyse_target_expression(env, self.classobj)
4435 4436
        self.class_cell = self.class_cell.analyse_expressions(cenv)
        return self
4437

4438
    def generate_function_definitions(self, env, code):
4439
        self.generate_lambda_definitions(self.scope, code)
4440
        self.body.generate_function_definitions(self.scope, code)
4441

William Stein's avatar
William Stein committed
4442
    def generate_execution_code(self, code):
4443
        code.mark_pos(self.pos)
4444 4445
        code.pyclass_stack.append(self)
        cenv = self.scope
4446
        if self.bases:
4447
            self.bases.generate_evaluation_code(code)
4448
        if self.mkw:
4449
            self.mkw.generate_evaluation_code(code)
4450
        if self.metaclass:
4451
            self.metaclass.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
4452
        self.dict.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4453
        cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
4454
        self.class_cell.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4455
        self.body.generate_execution_code(code)
4456
        self.class_result.generate_evaluation_code(code)
4457 4458 4459
        self.class_cell.generate_injection_code(
            code, self.class_result.result())
        self.class_cell.generate_disposal_code(code)
4460
        cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
4461
        self.target.generate_assignment_code(self.class_result, code)
William Stein's avatar
William Stein committed
4462
        self.dict.generate_disposal_code(code)
4463
        self.dict.free_temps(code)
4464
        if self.metaclass:
4465 4466
            self.metaclass.generate_disposal_code(code)
            self.metaclass.free_temps(code)
4467 4468 4469 4470
        if self.mkw:
            self.mkw.generate_disposal_code(code)
            self.mkw.free_temps(code)
        if self.bases:
4471 4472
            self.bases.generate_disposal_code(code)
            self.bases.free_temps(code)
4473
        code.pyclass_stack.pop()
William Stein's avatar
William Stein committed
4474

Robert Bradshaw's avatar
Robert Bradshaw committed
4475
class CClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4476 4477 4478 4479
    #  An extension type definition.
    #
    #  visibility         'private' or 'public' or 'extern'
    #  typedef_flag       boolean
Stefan Behnel's avatar
Stefan Behnel committed
4480
    #  api                boolean
William Stein's avatar
William Stein committed
4481 4482 4483 4484 4485 4486 4487 4488
    #  module_name        string or None    For import of extern type objects
    #  class_name         string            Unqualified name of class
    #  as_name            string or None    Name to declare as in this scope
    #  base_class_module  string or None    Module containing the base class
    #  base_class_name    string or None    Name of the base class
    #  objstruct_name     string or None    Specified C name of object struct
    #  typeobj_name       string or None    Specified C name of type object
    #  in_pxd             boolean           Is in a .pxd file
4489
    #  decorators         [DecoratorNode]   list of decorators or None
William Stein's avatar
William Stein committed
4490 4491 4492 4493
    #  doc                string or None
    #  body               StatNode or None
    #  entry              Symtab.Entry
    #  base_type          PyExtensionType or None
4494 4495
    #  buffer_defaults_node DictNode or None Declares defaults for a buffer
    #  buffer_defaults_pos
4496

4497
    child_attrs = ["body"]
4498 4499
    buffer_defaults_node = None
    buffer_defaults_pos = None
4500 4501 4502 4503
    typedef_flag = False
    api = False
    objstruct_name = None
    typeobj_name = None
4504
    decorators = None
4505
    shadow = False
4506

Robert Bradshaw's avatar
Robert Bradshaw committed
4507 4508
    def buffer_defaults(self, env):
        if not hasattr(self, '_buffer_defaults'):
4509
            from . import Buffer
Robert Bradshaw's avatar
Robert Bradshaw committed
4510 4511 4512 4513 4514 4515 4516 4517 4518
            if self.buffer_defaults_node:
                self._buffer_defaults = Buffer.analyse_buffer_options(
                    self.buffer_defaults_pos,
                    env, [], self.buffer_defaults_node,
                    need_complete=False)
            else:
                self._buffer_defaults = None
        return self._buffer_defaults

4519 4520 4521 4522 4523 4524 4525 4526
    def declare(self, env):
        if self.module_name and self.visibility != 'extern':
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return None
        else:
            home_scope = env
4527

4528
        self.entry = home_scope.declare_c_class(
4529 4530 4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541
            name=self.class_name,
            pos=self.pos,
            defining=0,
            implementing=0,
            module_name=self.module_name,
            base_type=None,
            objstruct_cname=self.objstruct_name,
            typeobj_cname=self.typeobj_name,
            visibility=self.visibility,
            typedef_flag=self.typedef_flag,
            api=self.api,
            buffer_defaults=self.buffer_defaults(env),
            shadow=self.shadow)
4542

William Stein's avatar
William Stein committed
4543 4544 4545 4546
    def analyse_declarations(self, env):
        #print "CClassDefNode.analyse_declarations:", self.class_name
        #print "...visibility =", self.visibility
        #print "...module_name =", self.module_name
4547

William Stein's avatar
William Stein committed
4548
        if env.in_cinclude and not self.objstruct_name:
4549
            error(self.pos, "Object struct name specification required for C class defined in 'extern from' block")
4550
        if self.decorators:
4551
            error(self.pos, "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
William Stein's avatar
William Stein committed
4552
        self.base_type = None
4553 4554
        # Now that module imports are cached, we need to
        # import the modules for extern classes.
4555 4556 4557 4558 4559 4560 4561
        if self.module_name:
            self.module = None
            for module in env.cimported_modules:
                if module.name == self.module_name:
                    self.module = module
            if self.module is None:
                self.module = ModuleScope(self.module_name, None, env.context)
4562
                self.module.has_extern_class = 1
Robert Bradshaw's avatar
Robert Bradshaw committed
4563
                env.add_imported_module(self.module)
4564

William Stein's avatar
William Stein committed
4565 4566
        if self.base_class_name:
            if self.base_class_module:
4567 4568 4569 4570
                base_class_scope = env.find_imported_module(self.base_class_module.split('.'), self.pos)
                if not base_class_scope:
                    error(self.pos, "'%s' is not a cimported module" % self.base_class_module)
                    return
William Stein's avatar
William Stein committed
4571 4572
            else:
                base_class_scope = env
4573 4574 4575 4576 4577 4578
            if self.base_class_name == 'object':
                # extension classes are special and don't need to inherit from object
                if base_class_scope is None or base_class_scope.lookup('object') is None:
                    self.base_class_name = None
                    self.base_class_module = None
                    base_class_scope = None
William Stein's avatar
William Stein committed
4579 4580 4581 4582 4583
            if base_class_scope:
                base_class_entry = base_class_scope.find(self.base_class_name, self.pos)
                if base_class_entry:
                    if not base_class_entry.is_type:
                        error(self.pos, "'%s' is not a type name" % self.base_class_name)
4584
                    elif not base_class_entry.type.is_extension_type and \
Stefan Behnel's avatar
Stefan Behnel committed
4585
                             not (base_class_entry.type.is_builtin_type and
4586
                                  base_class_entry.type.objstruct_cname):
William Stein's avatar
William Stein committed
4587 4588
                        error(self.pos, "'%s' is not an extension type" % self.base_class_name)
                    elif not base_class_entry.type.is_complete():
4589 4590 4591
                        error(self.pos, "Base class '%s' of type '%s' is incomplete" % (
                            self.base_class_name, self.class_name))
                    elif base_class_entry.type.scope and base_class_entry.type.scope.directives and \
4592
                             base_class_entry.type.is_final_type:
4593 4594
                        error(self.pos, "Base class '%s' of type '%s' is final" % (
                            self.base_class_name, self.class_name))
4595 4596 4597 4598
                    elif base_class_entry.type.is_builtin_type and \
                             base_class_entry.type.name in ('tuple', 'str', 'bytes'):
                        error(self.pos, "inheritance from PyVarObject types like '%s' is not currently supported"
                              % base_class_entry.type.name)
William Stein's avatar
William Stein committed
4599 4600
                    else:
                        self.base_type = base_class_entry.type
4601
                if env.directives.get('freelist', 0) > 0:
Stefan Behnel's avatar
Stefan Behnel committed
4602
                    warning(self.pos, "freelists cannot be used on subtypes, only the base class can manage them", 1)
4603

William Stein's avatar
William Stein committed
4604
        has_body = self.body is not None
4605 4606 4607 4608 4609 4610
        if has_body and self.base_type and not self.base_type.scope:
            # To properly initialize inherited attributes, the base type must
            # be analysed before this type.
            self.base_type.defered_declarations.append(lambda : self.analyse_declarations(env))
            return

4611
        if self.module_name and self.visibility != 'extern':
4612 4613 4614 4615 4616 4617
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return
        else:
            home_scope = env
4618 4619

        if self.visibility == 'extern':
4620
            if (self.module_name == '__builtin__' and
4621 4622
                    self.class_name in Builtin.builtin_types and
                    env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython
4623
                warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1)
4624

4625
        self.entry = home_scope.declare_c_class(
4626 4627 4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638
            name=self.class_name,
            pos=self.pos,
            defining=has_body and self.in_pxd,
            implementing=has_body and not self.in_pxd,
            module_name=self.module_name,
            base_type=self.base_type,
            objstruct_cname=self.objstruct_name,
            typeobj_cname=self.typeobj_name,
            visibility=self.visibility,
            typedef_flag=self.typedef_flag,
            api=self.api,
            buffer_defaults=self.buffer_defaults(env),
            shadow=self.shadow)
4639

4640 4641
        if self.shadow:
            home_scope.lookup(self.class_name).as_variable = self.entry
4642
        if home_scope is not env and self.visibility == 'extern':
4643
            env.add_imported_entry(self.class_name, self.entry, self.pos)
4644
        self.scope = scope = self.entry.type.scope
4645 4646
        if scope is not None:
            scope.directives = env.directives
4647

4648
        if self.doc and Options.docstrings:
4649
            scope.doc = embed_position(self.pos, self.doc)
4650

William Stein's avatar
William Stein committed
4651 4652
        if has_body:
            self.body.analyse_declarations(scope)
4653
            dict_entry = self.scope.lookup_here("__dict__")
4654
            if dict_entry and dict_entry.is_variable and (not scope.defined and not scope.implemented):
4655 4656
                dict_entry.getter_cname = self.scope.mangle_internal("__dict__getter")
                self.scope.declare_property("__dict__", dict_entry.doc, dict_entry.pos)
William Stein's avatar
William Stein committed
4657 4658 4659 4660
            if self.in_pxd:
                scope.defined = 1
            else:
                scope.implemented = 1
4661

William Stein's avatar
William Stein committed
4662
        env.allocate_vtable_names(self.entry)
4663

4664 4665 4666
        for thunk in self.entry.type.defered_declarations:
            thunk()

William Stein's avatar
William Stein committed
4667 4668
    def analyse_expressions(self, env):
        if self.body:
Robert Bradshaw's avatar
Robert Bradshaw committed
4669
            scope = self.entry.type.scope
4670 4671
            self.body = self.body.analyse_expressions(scope)
        return self
4672

4673
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
4674
        if self.body:
4675 4676
            self.generate_lambda_definitions(self.scope, code)
            self.body.generate_function_definitions(self.scope, code)
4677

William Stein's avatar
William Stein committed
4678 4679 4680
    def generate_execution_code(self, code):
        # This is needed to generate evaluation code for
        # default values of method arguments.
4681
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4682 4683
        if self.body:
            self.body.generate_execution_code(code)
4684

4685 4686 4687
    def annotate(self, code):
        if self.body:
            self.body.annotate(code)
William Stein's avatar
William Stein committed
4688 4689 4690 4691 4692 4693


class PropertyNode(StatNode):
    #  Definition of a property in an extension type.
    #
    #  name   string
4694
    #  doc    EncodedString or None    Doc string
4695
    #  entry  Symtab.Entry
William Stein's avatar
William Stein committed
4696
    #  body   StatListNode
4697

4698 4699
    child_attrs = ["body"]

William Stein's avatar
William Stein committed
4700
    def analyse_declarations(self, env):
4701 4702 4703
        self.entry = env.declare_property(self.name, self.doc, self.pos)
        self.entry.scope.directives = env.directives
        self.body.analyse_declarations(self.entry.scope)
4704

William Stein's avatar
William Stein committed
4705
    def analyse_expressions(self, env):
4706 4707
        self.body = self.body.analyse_expressions(env)
        return self
4708

4709 4710
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
4711 4712 4713 4714

    def generate_execution_code(self, code):
        pass

4715 4716 4717
    def annotate(self, code):
        self.body.annotate(code)

William Stein's avatar
William Stein committed
4718 4719 4720 4721 4722

class GlobalNode(StatNode):
    # Global variable declaration.
    #
    # names    [string]
4723

4724 4725
    child_attrs = []

William Stein's avatar
William Stein committed
4726 4727 4728 4729 4730
    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_global(name, self.pos)

    def analyse_expressions(self, env):
4731
        return self
4732

William Stein's avatar
William Stein committed
4733 4734 4735 4736
    def generate_execution_code(self, code):
        pass


4737 4738 4739 4740 4741 4742 4743 4744 4745 4746 4747 4748
class NonlocalNode(StatNode):
    # Nonlocal variable declaration via the 'nonlocal' keyword.
    #
    # names    [string]

    child_attrs = []

    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_nonlocal(name, self.pos)

    def analyse_expressions(self, env):
4749
        return self
4750 4751 4752 4753 4754

    def generate_execution_code(self, code):
        pass


William Stein's avatar
William Stein committed
4755 4756 4757 4758
class ExprStatNode(StatNode):
    #  Expression used as a statement.
    #
    #  expr   ExprNode
4759 4760

    child_attrs = ["expr"]
4761

Robert Bradshaw's avatar
Robert Bradshaw committed
4762
    def analyse_declarations(self, env):
4763
        from . import ExprNodes
Robert Bradshaw's avatar
Robert Bradshaw committed
4764
        if isinstance(self.expr, ExprNodes.GeneralCallNode):
4765
            func = self.expr.function.as_cython_attribute()
Robert Bradshaw's avatar
Robert Bradshaw committed
4766 4767 4768 4769 4770 4771 4772 4773 4774
            if func == u'declare':
                args, kwds = self.expr.explicit_args_kwds()
                if len(args):
                    error(self.expr.pos, "Variable names must be specified.")
                for var, type_node in kwds.key_value_pairs:
                    type = type_node.analyse_as_type(env)
                    if type is None:
                        error(type_node.pos, "Unknown type")
                    else:
4775
                        env.declare_var(var.value, type, var.pos, is_cdef=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
4776
                self.__class__ = PassStatNode
4777

William Stein's avatar
William Stein committed
4778
    def analyse_expressions(self, env):
4779
        self.expr.result_is_used = False  # hint that .result() may safely be left empty
4780 4781
        self.expr = self.expr.analyse_expressions(env)
        return self
4782

4783
    def nogil_check(self, env):
4784
        if self.expr.type.is_pyobject and self.expr.is_temp:
4785 4786 4787 4788
            self.gil_error()

    gil_message = "Discarding owned Python object"

William Stein's avatar
William Stein committed
4789
    def generate_execution_code(self, code):
4790
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4791
        self.expr.generate_evaluation_code(code)
4792 4793
        if not self.expr.is_temp and self.expr.result():
            code.putln("%s;" % self.expr.result())
William Stein's avatar
William Stein committed
4794
        self.expr.generate_disposal_code(code)
4795
        self.expr.free_temps(code)
William Stein's avatar
William Stein committed
4796

4797 4798 4799
    def generate_function_definitions(self, env, code):
        self.expr.generate_function_definitions(env, code)

4800 4801 4802
    def annotate(self, code):
        self.expr.annotate(code)

William Stein's avatar
William Stein committed
4803 4804 4805 4806 4807 4808 4809 4810 4811 4812 4813

class AssignmentNode(StatNode):
    #  Abstract base class for assignment nodes.
    #
    #  The analyse_expressions and generate_execution_code
    #  phases of assignments are split into two sub-phases
    #  each, to enable all the right hand sides of a
    #  parallel assignment to be evaluated before assigning
    #  to any of the left hand sides.

    def analyse_expressions(self, env):
4814
        node = self.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
4815
        if isinstance(node, AssignmentNode) and not isinstance(node, ParallelAssignmentNode):
4816
            if node.rhs.type.is_ptr and node.rhs.is_ephemeral():
4817 4818
                error(self.pos, "Storing unsafe C derivative of temporary Python reference")
        return node
4819

4820 4821 4822
#       def analyse_expressions(self, env):
#           self.analyse_expressions_1(env)
#           self.analyse_expressions_2(env)
William Stein's avatar
William Stein committed
4823 4824

    def generate_execution_code(self, code):
4825
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4826 4827
        self.generate_rhs_evaluation_code(code)
        self.generate_assignment_code(code)
4828

William Stein's avatar
William Stein committed
4829 4830 4831 4832 4833 4834

class SingleAssignmentNode(AssignmentNode):
    #  The simplest case:
    #
    #    a = b
    #
4835 4836 4837 4838
    #  lhs                      ExprNode      Left hand side
    #  rhs                      ExprNode      Right hand side
    #  first                    bool          Is this guaranteed the first assignment to lhs?
    #  is_overloaded_assignment bool          Is this assignment done via an overloaded operator=
4839 4840
    #  exception_check
    #  exception_value
4841

4842
    child_attrs = ["lhs", "rhs"]
4843
    first = False
4844
    is_overloaded_assignment = False
4845
    declaration_only = False
William Stein's avatar
William Stein committed
4846 4847

    def analyse_declarations(self, env):
4848
        from . import ExprNodes
4849

4850 4851
        # handle declarations of the form x = cython.foo()
        if isinstance(self.rhs, ExprNodes.CallNode):
4852
            func_name = self.rhs.function.as_cython_attribute()
4853 4854 4855
            if func_name:
                args, kwds = self.rhs.explicit_args_kwds()
                if func_name in ['declare', 'typedef']:
4856 4857
                    if len(args) > 2:
                        error(args[2].pos, "Invalid positional argument.")
4858
                        return
4859 4860 4861 4862 4863 4864 4865 4866
                    if kwds is not None:
                        kwdict = kwds.compile_time_value(None)
                        if func_name == 'typedef' or 'visibility' not in kwdict:
                            error(kwds.pos, "Invalid keyword argument.")
                            return
                        visibility = kwdict['visibility']
                    else:
                        visibility = 'private'
4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880
                    type = args[0].analyse_as_type(env)
                    if type is None:
                        error(args[0].pos, "Unknown type")
                        return
                    lhs = self.lhs
                    if func_name == 'declare':
                        if isinstance(lhs, ExprNodes.NameNode):
                            vars = [(lhs.name, lhs.pos)]
                        elif isinstance(lhs, ExprNodes.TupleNode):
                            vars = [(var.name, var.pos) for var in lhs.args]
                        else:
                            error(lhs.pos, "Invalid declaration")
                            return
                        for var, pos in vars:
4881
                            env.declare_var(var, type, pos, is_cdef=True, visibility=visibility)
Robert Bradshaw's avatar
Robert Bradshaw committed
4882 4883 4884 4885 4886
                        if len(args) == 2:
                            # we have a value
                            self.rhs = args[1]
                        else:
                            self.declaration_only = True
4887
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
4888
                        self.declaration_only = True
4889 4890
                        if not isinstance(lhs, ExprNodes.NameNode):
                            error(lhs.pos, "Invalid declaration.")
4891
                        env.declare_typedef(lhs.name, type, self.pos, visibility='private')
4892

4893 4894 4895
                elif func_name in ['struct', 'union']:
                    self.declaration_only = True
                    if len(args) > 0 or kwds is None:
4896
                        error(self.rhs.pos, "Struct or union members must be given by name.")
4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913
                        return
                    members = []
                    for member, type_node in kwds.key_value_pairs:
                        type = type_node.analyse_as_type(env)
                        if type is None:
                            error(type_node.pos, "Unknown type")
                        else:
                            members.append((member.value, type, member.pos))
                    if len(members) < len(kwds.key_value_pairs):
                        return
                    if not isinstance(self.lhs, ExprNodes.NameNode):
                        error(self.lhs.pos, "Invalid declaration.")
                    name = self.lhs.name
                    scope = StructOrUnionScope(name)
                    env.declare_struct_or_union(name, func_name, scope, False, self.rhs.pos)
                    for member, type, pos in members:
                        scope.declare_var(member, type, pos)
4914

Mark Florisson's avatar
Mark Florisson committed
4915 4916 4917 4918 4919 4920 4921 4922
                elif func_name == 'fused_type':
                    # dtype = cython.fused_type(...)
                    self.declaration_only = True
                    if kwds:
                        error(self.rhs.function.pos,
                              "fused_type does not take keyword arguments")

                    fusednode = FusedTypeNode(self.rhs.pos,
4923
                                              name=self.lhs.name, types=args)
Mark Florisson's avatar
Mark Florisson committed
4924 4925
                    fusednode.analyse_declarations(env)

4926 4927 4928 4929
        if self.declaration_only:
            return
        else:
            self.lhs.analyse_target_declaration(env)
4930

4931 4932
    def analyse_types(self, env, use_temp=0):
        from . import ExprNodes
4933

4934
        self.rhs = self.rhs.analyse_types(env)
4935

4936 4937 4938
        unrolled_assignment = self.unroll_rhs(env)
        if unrolled_assignment:
            return unrolled_assignment
Robert Bradshaw's avatar
Robert Bradshaw committed
4939

4940
        self.lhs = self.lhs.analyse_target_types(env)
4941
        self.lhs.gil_assignment_check(env)
4942 4943 4944
        unrolled_assignment = self.unroll_lhs(env)
        if unrolled_assignment:
            return unrolled_assignment
4945

4946 4947 4948
        if isinstance(self.lhs, ExprNodes.MemoryViewIndexNode):
            self.lhs.analyse_broadcast_operation(self.rhs)
            self.lhs = self.lhs.analyse_as_memview_scalar_assignment(self.rhs)
4949
        elif self.lhs.type.is_array:
4950
            if not isinstance(self.lhs, ExprNodes.SliceIndexNode):
4951
                # cannot assign to C array, only to its full slice
4952
                self.lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None)
4953
                self.lhs = self.lhs.analyse_target_types(env)
4954

4955 4956 4957 4958
        if self.lhs.type.is_cpp_class:
            op = env.lookup_operator_for_types(self.pos, '=', [self.lhs.type, self.rhs.type])
            if op:
                rhs = self.rhs
4959
                self.is_overloaded_assignment = True
4960 4961 4962 4963
                self.exception_check = op.type.exception_check
                self.exception_value = op.type.exception_value
                if self.exception_check == '+' and self.exception_value is None:
                    env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp"))
4964
            else:
4965
                rhs = self.rhs.coerce_to(self.lhs.type, env)
4966
        else:
4967 4968
            rhs = self.rhs.coerce_to(self.lhs.type, env)

4969 4970 4971
        if use_temp or rhs.is_attribute or (
                not rhs.is_name and not rhs.is_literal and
                rhs.type.is_pyobject):
4972
            # things like (cdef) attribute access are not safe (traverses pointers)
4973 4974 4975 4976
            rhs = rhs.coerce_to_temp(env)
        elif rhs.type.is_pyobject:
            rhs = rhs.coerce_to_simple(env)
        self.rhs = rhs
4977
        return self
4978

4979 4980
    def unroll(self, node, target_size, env):
        from . import ExprNodes, UtilNodes
4981 4982 4983 4984

        base = node
        start_node = stop_node = step_node = check_node = None

4985
        if node.type.is_ctuple:
4986
            slice_size = node.type.size
4987

4988 4989 4990
        elif node.type.is_ptr or node.type.is_array:
            while isinstance(node, ExprNodes.SliceIndexNode) and not (node.start or node.stop):
                base = node = node.base
4991 4992 4993
            if isinstance(node, ExprNodes.SliceIndexNode):
                base = node.base
                start_node = node.start
4994 4995
                if start_node:
                    start_node = start_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
4996
                stop_node = node.stop
4997 4998 4999
                if stop_node:
                    stop_node = stop_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
                else:
5000
                    if node.type.is_array and node.type.size:
5001 5002
                        stop_node = ExprNodes.IntNode(
                            self.pos, value=str(node.type.size),
5003
                            constant_result=(node.type.size if isinstance(node.type.size, _py_int_types)
5004
                                             else ExprNodes.constant_value_not_set))
5005 5006 5007
                    else:
                        error(self.pos, "C array iteration requires known end index")
                        return
5008
                step_node = None  #node.step
5009 5010
                if step_node:
                    step_node = step_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5011

5012 5013 5014 5015
                # TODO: Factor out SliceIndexNode.generate_slice_guard_code() for use here.
                def get_const(node, none_value):
                    if node is None:
                        return none_value
5016
                    elif node.has_constant_result():
5017 5018
                        return node.constant_result
                    else:
5019 5020
                        raise ValueError("Not a constant.")

5021 5022 5023 5024 5025
                try:
                    slice_size = (get_const(stop_node, None) - get_const(start_node, 0)) / get_const(step_node, 1)
                except ValueError:
                    error(self.pos, "C array assignment currently requires known endpoints")
                    return
5026 5027 5028

            elif node.type.is_array:
                slice_size = node.type.size
5029
                if not isinstance(slice_size, _py_int_types):
5030
                    return  # might still work when coercing to Python
5031 5032 5033 5034 5035 5036
            else:
                return

        else:
            return

5037 5038 5039 5040 5041
        if slice_size != target_size:
            error(self.pos, "Assignment to/from slice of wrong length, expected %s, got %s" % (
                slice_size, target_size))
            return

5042
        items = []
5043 5044 5045
        base = UtilNodes.LetRefNode(base)
        refs = [base]
        if start_node and not start_node.is_literal:
5046 5047
            start_node = UtilNodes.LetRefNode(start_node)
            refs.append(start_node)
5048
        if stop_node and not stop_node.is_literal:
5049 5050
            stop_node = UtilNodes.LetRefNode(stop_node)
            refs.append(stop_node)
5051
        if step_node and not step_node.is_literal:
5052 5053
            step_node = UtilNodes.LetRefNode(step_node)
            refs.append(step_node)
5054

5055
        for ix in range(target_size):
5056
            ix_node = ExprNodes.IntNode(self.pos, value=str(ix), constant_result=ix, type=PyrexTypes.c_py_ssize_t_type)
5057
            if step_node is not None:
5058 5059 5060 5061 5062
                if step_node.has_constant_result():
                    step_value = ix_node.constant_result * step_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(step_value), constant_result=step_value)
                else:
                    ix_node = ExprNodes.MulNode(self.pos, operator='*', operand1=step_node, operand2=ix_node)
5063
            if start_node is not None:
5064 5065 5066 5067 5068 5069 5070
                if start_node.has_constant_result() and ix_node.has_constant_result():
                    index_value = ix_node.constant_result + start_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(index_value), constant_result=index_value)
                else:
                    ix_node = ExprNodes.AddNode(
                        self.pos, operator='+', operand1=start_node, operand2=ix_node)
            items.append(ExprNodes.IndexNode(self.pos, base=base, index=ix_node.analyse_types(env)))
5071 5072
        return check_node, refs, items

5073
    def unroll_assignments(self, refs, check_node, lhs_list, rhs_list, env):
5074
        from . import UtilNodes
5075
        assignments = []
5076
        for lhs, rhs in zip(lhs_list, rhs_list):
5077
            assignments.append(SingleAssignmentNode(self.pos, lhs=lhs, rhs=rhs, first=self.first))
5078
        node = ParallelAssignmentNode(pos=self.pos, stats=assignments).analyse_expressions(env)
5079
        if check_node:
5080
            node = StatListNode(pos=self.pos, stats=[check_node, node])
5081
        for ref in refs[::-1]:
5082 5083
            node = UtilNodes.LetNode(ref, node)
        return node
5084

5085
    def unroll_rhs(self, env):
5086
        from . import ExprNodes
5087 5088
        if not isinstance(self.lhs, ExprNodes.TupleNode):
            return
5089 5090
        if any(arg.is_starred for arg in self.lhs.args):
            return
5091 5092 5093 5094 5095 5096 5097 5098 5099 5100 5101

        unrolled = self.unroll(self.rhs, len(self.lhs.args), env)
        if not unrolled:
            return
        check_node, refs, rhs = unrolled
        return self.unroll_assignments(refs, check_node, self.lhs.args, rhs, env)

    def unroll_lhs(self, env):
        if self.lhs.type.is_ctuple:
            # Handled directly.
            return
5102
        from . import ExprNodes
5103 5104 5105 5106 5107 5108 5109 5110 5111
        if not isinstance(self.rhs, ExprNodes.TupleNode):
            return

        unrolled = self.unroll(self.lhs, len(self.rhs.args), env)
        if not unrolled:
            return
        check_node, refs, lhs = unrolled
        return self.unroll_assignments(refs, check_node, lhs, self.rhs.args, env)

William Stein's avatar
William Stein committed
5112 5113
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5114

5115
    def generate_assignment_code(self, code, overloaded_assignment=False):
5116 5117 5118 5119 5120 5121 5122 5123 5124
        if self.is_overloaded_assignment:
            self.lhs.generate_assignment_code(
                self.rhs,
                code,
                overloaded_assignment=self.is_overloaded_assignment,
                exception_check=self.exception_check,
                exception_value=self.exception_value)
        else:
            self.lhs.generate_assignment_code(self.rhs, code)
William Stein's avatar
William Stein committed
5125

5126 5127 5128
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5129 5130 5131 5132
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

William Stein's avatar
William Stein committed
5133 5134 5135 5136 5137 5138 5139 5140 5141 5142 5143

class CascadedAssignmentNode(AssignmentNode):
    #  An assignment with multiple left hand sides:
    #
    #    a = b = c
    #
    #  lhs_list   [ExprNode]   Left hand sides
    #  rhs        ExprNode     Right hand sides
    #
    #  Used internally:
    #
5144 5145 5146
    #  coerced_values       [ExprNode]   RHS coerced to all distinct LHS types
    #  cloned_values        [ExprNode]   cloned RHS value for each LHS
    #  assignment_overloads [Bool]       If each assignment uses a C++ operator=
5147

5148 5149
    child_attrs = ["lhs_list", "rhs", "coerced_values", "cloned_values"]
    cloned_values = None
5150
    coerced_values = None
5151
    assignment_overloads = None
5152

William Stein's avatar
William Stein committed
5153 5154 5155
    def analyse_declarations(self, env):
        for lhs in self.lhs_list:
            lhs.analyse_target_declaration(env)
5156

5157
    def analyse_types(self, env, use_temp=0):
5158
        from .ExprNodes import CloneNode, ProxyNode
5159

5160
        # collect distinct types used on the LHS
5161
        lhs_types = set()
5162 5163
        for i, lhs in enumerate(self.lhs_list):
            lhs = self.lhs_list[i] = lhs.analyse_target_types(env)
5164 5165 5166
            lhs.gil_assignment_check(env)
            lhs_types.add(lhs.type)

5167
        rhs = self.rhs.analyse_types(env)
5168
        # common special case: only one type needed on the LHS => coerce only once
5169
        if len(lhs_types) == 1:
5170 5171 5172 5173 5174 5175 5176
            # Avoid coercion for overloaded assignment operators.
            if next(iter(lhs_types)).is_cpp_class:
                op = env.lookup_operator('=', [lhs, self.rhs])
                if not op:
                    rhs = rhs.coerce_to(lhs_types.pop(), env)
            else:
                rhs = rhs.coerce_to(lhs_types.pop(), env)
5177 5178 5179

        if not rhs.is_name and not rhs.is_literal and (
                use_temp or rhs.is_attribute or rhs.type.is_pyobject):
5180
            rhs = rhs.coerce_to_temp(env)
5181
        else:
5182
            rhs = rhs.coerce_to_simple(env)
5183 5184
        self.rhs = ProxyNode(rhs) if rhs.is_temp else rhs

5185
        # clone RHS and coerce it to all distinct LHS types
5186 5187
        self.coerced_values = []
        coerced_values = {}
5188
        self.assignment_overloads = []
5189
        for lhs in self.lhs_list:
5190 5191
            overloaded = lhs.type.is_cpp_class and env.lookup_operator('=', [lhs, self.rhs])
            self.assignment_overloads.append(overloaded)
5192
            if lhs.type not in coerced_values and lhs.type != rhs.type:
5193
                rhs = CloneNode(self.rhs)
5194
                if not overloaded:
5195
                    rhs = rhs.coerce_to(lhs.type, env)
5196 5197
                self.coerced_values.append(rhs)
                coerced_values[lhs.type] = rhs
5198

5199 5200
        # clone coerced values for all LHS assignments
        self.cloned_values = []
William Stein's avatar
William Stein committed
5201
        for lhs in self.lhs_list:
5202
            rhs = coerced_values.get(lhs.type, self.rhs)
5203
            self.cloned_values.append(CloneNode(rhs))
5204
        return self
5205

William Stein's avatar
William Stein committed
5206 5207
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5208

5209
    def generate_assignment_code(self, code, overloaded_assignment=False):
5210
        # prepare all coercions
5211 5212
        for rhs in self.coerced_values:
            rhs.generate_evaluation_code(code)
5213
        # assign clones to LHS
5214
        for lhs, rhs, overload in zip(self.lhs_list, self.cloned_values, self.assignment_overloads):
William Stein's avatar
William Stein committed
5215
            rhs.generate_evaluation_code(code)
5216
            lhs.generate_assignment_code(rhs, code, overloaded_assignment=overload)
5217 5218 5219 5220
        # dispose of coerced values and original RHS
        for rhs_value in self.coerced_values:
            rhs_value.generate_disposal_code(code)
            rhs_value.free_temps(code)
William Stein's avatar
William Stein committed
5221
        self.rhs.generate_disposal_code(code)
5222
        self.rhs.free_temps(code)
William Stein's avatar
William Stein committed
5223

5224 5225 5226
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5227
    def annotate(self, code):
5228 5229
        for rhs in self.coerced_values:
            rhs.annotate(code)
5230
        for lhs, rhs in zip(self.lhs_list, self.cloned_values):
5231 5232
            lhs.annotate(code)
            rhs.annotate(code)
5233
        self.rhs.annotate(code)
5234

5235

William Stein's avatar
William Stein committed
5236 5237 5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248
class ParallelAssignmentNode(AssignmentNode):
    #  A combined packing/unpacking assignment:
    #
    #    a, b, c =  d, e, f
    #
    #  This has been rearranged by the parser into
    #
    #    a = d ; b = e ; c = f
    #
    #  but we must evaluate all the right hand sides
    #  before assigning to any of the left hand sides.
    #
    #  stats     [AssignmentNode]   The constituent assignments
5249

5250 5251
    child_attrs = ["stats"]

William Stein's avatar
William Stein committed
5252 5253 5254
    def analyse_declarations(self, env):
        for stat in self.stats:
            stat.analyse_declarations(env)
5255

William Stein's avatar
William Stein committed
5256
    def analyse_expressions(self, env):
5257 5258
        self.stats = [stat.analyse_types(env, use_temp=1)
                      for stat in self.stats]
5259
        return self
5260

Robert Bradshaw's avatar
Robert Bradshaw committed
5261 5262
#    def analyse_expressions(self, env):
#        for stat in self.stats:
5263
#            stat.analyse_expressions_1(env, use_temp=1)
Robert Bradshaw's avatar
Robert Bradshaw committed
5264 5265
#        for stat in self.stats:
#            stat.analyse_expressions_2(env)
5266

William Stein's avatar
William Stein committed
5267
    def generate_execution_code(self, code):
5268
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5269 5270 5271 5272 5273
        for stat in self.stats:
            stat.generate_rhs_evaluation_code(code)
        for stat in self.stats:
            stat.generate_assignment_code(code)

5274 5275 5276 5277
    def generate_function_definitions(self, env, code):
        for stat in self.stats:
            stat.generate_function_definitions(env, code)

5278 5279 5280 5281 5282
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)


5283
class InPlaceAssignmentNode(AssignmentNode):
Craig Citro's avatar
Craig Citro committed
5284
    #  An in place arithmetic operand:
5285 5286 5287 5288 5289 5290 5291
    #
    #    a += b
    #    a -= b
    #    ...
    #
    #  lhs      ExprNode      Left hand side
    #  rhs      ExprNode      Right hand side
Stefan Behnel's avatar
Stefan Behnel committed
5292
    #  operator char          one of "+-*/%^&|"
5293
    #
5294 5295 5296 5297 5298 5299 5300
    #  This code is a bit tricky because in order to obey Python
    #  semantics the sub-expressions (e.g. indices) of the lhs must
    #  not be evaluated twice. So we must re-use the values calculated
    #  in evaluation phase for the assignment phase as well.
    #  Fortunately, the type of the lhs node is fairly constrained
    #  (it must be a NameNode, AttributeNode, or IndexNode).

5301
    child_attrs = ["lhs", "rhs"]
5302

5303 5304
    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)
5305

5306
    def analyse_types(self, env):
5307 5308
        self.rhs = self.rhs.analyse_types(env)
        self.lhs = self.lhs.analyse_target_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5309

5310
        # When assigning to a fully indexed buffer or memoryview, coerce the rhs
5311
        if self.lhs.is_memview_index or self.lhs.is_buffer_access:
5312
            self.rhs = self.rhs.coerce_to(self.lhs.type, env)
5313 5314 5315
        elif self.lhs.type.is_string and self.operator in '+-':
            # use pointer arithmetic for char* LHS instead of string concat
            self.rhs = self.rhs.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5316
        return self
5317

5318
    def generate_execution_code(self, code):
5319
        code.mark_pos(self.pos)
5320 5321 5322
        lhs, rhs = self.lhs, self.rhs
        rhs.generate_evaluation_code(code)
        lhs.generate_subexpr_evaluation_code(code)
5323 5324 5325 5326 5327
        c_op = self.operator
        if c_op == "//":
            c_op = "/"
        elif c_op == "**":
            error(self.pos, "No C inplace power operator")
5328 5329
        if lhs.is_buffer_access or lhs.is_memview_index:
            if lhs.type.is_pyobject:
5330
                error(self.pos, "In-place operators not allowed on object buffers in this release.")
5331
            if c_op in ('/', '%') and lhs.type.is_int and not code.globalstate.directives['cdivision']:
Robert Bradshaw's avatar
Robert Bradshaw committed
5332
                error(self.pos, "In-place non-c divide operators not allowed on int buffers.")
5333 5334 5335
            lhs.generate_buffer_setitem_code(rhs, code, c_op)
        elif lhs.is_memview_slice:
            error(self.pos, "Inplace operators not supported on memoryview slices")
5336 5337 5338
        else:
            # C++
            # TODO: make sure overload is declared
5339 5340 5341 5342 5343
            code.putln("%s %s= %s;" % (lhs.result(), c_op, rhs.result()))
        lhs.generate_subexpr_disposal_code(code)
        lhs.free_subexpr_temps(code)
        rhs.generate_disposal_code(code)
        rhs.free_temps(code)
5344

5345 5346 5347
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)
5348

5349
    def create_binop_node(self):
5350
        from . import ExprNodes
5351
        return ExprNodes.binop_node(self.pos, self.operator, self.lhs, self.rhs)
5352

William Stein's avatar
William Stein committed
5353 5354 5355 5356

class PrintStatNode(StatNode):
    #  print statement
    #
5357
    #  arg_tuple         TupleNode
5358
    #  stream            ExprNode or None (stdout)
5359
    #  append_newline    boolean
5360

5361
    child_attrs = ["arg_tuple", "stream"]
5362

William Stein's avatar
William Stein committed
5363
    def analyse_expressions(self, env):
5364
        if self.stream:
5365 5366 5367 5368
            stream = self.stream.analyse_expressions(env)
            self.stream = stream.coerce_to_pyobject(env)
        arg_tuple = self.arg_tuple.analyse_expressions(env)
        self.arg_tuple = arg_tuple.coerce_to_pyobject(env)
5369
        env.use_utility_code(printing_utility_code)
5370 5371
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            env.use_utility_code(printing_one_utility_code)
5372
        return self
5373

5374
    nogil_check = Node.gil_error
5375
    gil_message = "Python print statement"
5376

William Stein's avatar
William Stein committed
5377
    def generate_execution_code(self, code):
5378
        code.mark_pos(self.pos)
5379 5380 5381 5382 5383
        if self.stream:
            self.stream.generate_evaluation_code(code)
            stream_result = self.stream.py_result()
        else:
            stream_result = '0'
5384 5385 5386
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            arg = self.arg_tuple.args[0]
            arg.generate_evaluation_code(code)
5387

5388
            code.putln(
5389 5390
                "if (__Pyx_PrintOne(%s, %s) < 0) %s" % (
                    stream_result,
5391 5392 5393 5394 5395 5396 5397
                    arg.py_result(),
                    code.error_goto(self.pos)))
            arg.generate_disposal_code(code)
            arg.free_temps(code)
        else:
            self.arg_tuple.generate_evaluation_code(code)
            code.putln(
5398 5399
                "if (__Pyx_Print(%s, %s, %d) < 0) %s" % (
                    stream_result,
5400 5401 5402 5403 5404
                    self.arg_tuple.py_result(),
                    self.append_newline,
                    code.error_goto(self.pos)))
            self.arg_tuple.generate_disposal_code(code)
            self.arg_tuple.free_temps(code)
5405

5406 5407 5408 5409
        if self.stream:
            self.stream.generate_disposal_code(code)
            self.stream.free_temps(code)

5410
    def generate_function_definitions(self, env, code):
5411 5412
        if self.stream:
            self.stream.generate_function_definitions(env, code)
5413
        self.arg_tuple.generate_function_definitions(env, code)
5414

5415
    def annotate(self, code):
5416 5417
        if self.stream:
            self.stream.annotate(code)
5418
        self.arg_tuple.annotate(code)
William Stein's avatar
William Stein committed
5419 5420


5421 5422 5423 5424 5425 5426 5427 5428 5429
class ExecStatNode(StatNode):
    #  exec statement
    #
    #  args     [ExprNode]

    child_attrs = ["args"]

    def analyse_expressions(self, env):
        for i, arg in enumerate(self.args):
5430
            arg = arg.analyse_expressions(env)
5431 5432 5433
            arg = arg.coerce_to_pyobject(env)
            self.args[i] = arg
        env.use_utility_code(Builtin.pyexec_utility_code)
5434
        return self
5435

5436
    nogil_check = Node.gil_error
5437 5438 5439
    gil_message = "Python exec statement"

    def generate_execution_code(self, code):
5440
        code.mark_pos(self.pos)
5441 5442 5443
        args = []
        for arg in self.args:
            arg.generate_evaluation_code(code)
5444
            args.append(arg.py_result())
5445
        args = tuple(args + ['0', '0'][:3-len(args)])
5446
        temp_result = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
5447
        code.putln("%s = __Pyx_PyExec3(%s, %s, %s);" % ((temp_result,) + args))
5448 5449
        for arg in self.args:
            arg.generate_disposal_code(code)
5450
            arg.free_temps(code)
5451
        code.putln(
5452 5453 5454 5455
            code.error_goto_if_null(temp_result, self.pos))
        code.put_gotref(temp_result)
        code.put_decref_clear(temp_result, py_object_type)
        code.funcstate.release_temp(temp_result)
5456 5457 5458 5459 5460 5461

    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)


William Stein's avatar
William Stein committed
5462 5463 5464 5465
class DelStatNode(StatNode):
    #  del statement
    #
    #  args     [ExprNode]
5466

5467
    child_attrs = ["args"]
5468
    ignore_nonexisting = False
5469

William Stein's avatar
William Stein committed
5470 5471 5472
    def analyse_declarations(self, env):
        for arg in self.args:
            arg.analyse_target_declaration(env)
5473

William Stein's avatar
William Stein committed
5474
    def analyse_expressions(self, env):
5475 5476
        for i, arg in enumerate(self.args):
            arg = self.args[i] = arg.analyse_target_expression(env, None)
5477
            if arg.type.is_pyobject or (arg.is_name and arg.type.is_memoryviewslice):
5478 5479
                if arg.is_name and arg.entry.is_cglobal:
                    error(arg.pos, "Deletion of global C variable")
Robert Bradshaw's avatar
Robert Bradshaw committed
5480
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
Robert Bradshaw's avatar
Robert Bradshaw committed
5481
                self.cpp_check(env)
5482
            elif arg.type.is_cpp_class:
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5483
                error(arg.pos, "Deletion of non-heap C++ object")
5484 5485
            elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
                pass  # del ba[i]
5486
            else:
Robert Bradshaw's avatar
Robert Bradshaw committed
5487
                error(arg.pos, "Deletion of non-Python, non-C++ object")
5488
            #arg.release_target_temp(env)
5489
        return self
5490

5491
    def nogil_check(self, env):
5492 5493
        for arg in self.args:
            if arg.type.is_pyobject:
5494
                self.gil_error()
5495

5496 5497
    gil_message = "Deleting Python object"

William Stein's avatar
William Stein committed
5498
    def generate_execution_code(self, code):
5499
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5500
        for arg in self.args:
5501 5502 5503
            if (arg.type.is_pyobject or
                    arg.type.is_memoryviewslice or
                    arg.is_subscript and arg.base.type is Builtin.bytearray_type):
5504 5505
                arg.generate_deletion_code(
                    code, ignore_nonexisting=self.ignore_nonexisting)
5506
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
5507
                arg.generate_evaluation_code(code)
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5508
                code.putln("delete %s;" % arg.result())
5509
                arg.generate_disposal_code(code)
William Stein's avatar
William Stein committed
5510 5511
            # else error reported earlier

5512 5513 5514 5515
    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)

William Stein's avatar
William Stein committed
5516 5517 5518

class PassStatNode(StatNode):
    #  pass statement
5519 5520

    child_attrs = []
5521

William Stein's avatar
William Stein committed
5522
    def analyse_expressions(self, env):
5523
        return self
5524

William Stein's avatar
William Stein committed
5525 5526 5527 5528
    def generate_execution_code(self, code):
        pass


5529 5530 5531 5532 5533 5534 5535 5536 5537
class IndirectionNode(StatListNode):
    """
    This adds an indirection so that the node can be shared and a subtree can
    be removed at any time by clearing self.stats.
    """

    def __init__(self, stats):
        super(IndirectionNode, self).__init__(stats[0].pos, stats=stats)

5538

William Stein's avatar
William Stein committed
5539 5540
class BreakStatNode(StatNode):

5541
    child_attrs = []
5542
    is_terminator = True
5543

William Stein's avatar
William Stein committed
5544
    def analyse_expressions(self, env):
5545
        return self
5546

William Stein's avatar
William Stein committed
5547
    def generate_execution_code(self, code):
5548
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5549 5550 5551
        if not code.break_label:
            error(self.pos, "break statement not inside loop")
        else:
5552
            code.put_goto(code.break_label)
William Stein's avatar
William Stein committed
5553 5554 5555 5556


class ContinueStatNode(StatNode):

5557
    child_attrs = []
5558
    is_terminator = True
5559

William Stein's avatar
William Stein committed
5560
    def analyse_expressions(self, env):
5561
        return self
5562

William Stein's avatar
William Stein committed
5563
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
5564
        if not code.continue_label:
William Stein's avatar
William Stein committed
5565
            error(self.pos, "continue statement not inside loop")
Stefan Behnel's avatar
Stefan Behnel committed
5566 5567 5568
            return
        code.mark_pos(self.pos)
        code.put_goto(code.continue_label)
William Stein's avatar
William Stein committed
5569 5570 5571 5572 5573 5574 5575


class ReturnStatNode(StatNode):
    #  return statement
    #
    #  value         ExprNode or None
    #  return_type   PyrexType
5576
    #  in_generator  return inside of generator => raise StopIteration
5577

5578
    child_attrs = ["value"]
5579
    is_terminator = True
5580
    in_generator = False
5581

5582 5583 5584
    # Whether we are in a parallel section
    in_parallel = False

William Stein's avatar
William Stein committed
5585 5586 5587 5588 5589
    def analyse_expressions(self, env):
        return_type = env.return_type
        self.return_type = return_type
        if not return_type:
            error(self.pos, "Return not inside a function body")
5590
            return self
William Stein's avatar
William Stein committed
5591
        if self.value:
5592
            self.value = self.value.analyse_types(env)
William Stein's avatar
William Stein committed
5593
            if return_type.is_void or return_type.is_returncode:
5594
                error(self.value.pos, "Return with value in void function")
William Stein's avatar
William Stein committed
5595 5596 5597 5598
            else:
                self.value = self.value.coerce_to(env.return_type, env)
        else:
            if (not return_type.is_void
5599 5600 5601
                    and not return_type.is_pyobject
                    and not return_type.is_returncode):
                error(self.pos, "Return value required")
5602
        return self
5603

5604
    def nogil_check(self, env):
5605
        if self.return_type.is_pyobject:
5606
            self.gil_error()
5607 5608 5609

    gil_message = "Returning Python object"

William Stein's avatar
William Stein committed
5610
    def generate_execution_code(self, code):
5611
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5612 5613 5614
        if not self.return_type:
            # error reported earlier
            return
5615 5616 5617
        if self.return_type.is_pyobject:
            code.put_xdecref(Naming.retval_cname,
                             self.return_type)
5618

William Stein's avatar
William Stein committed
5619 5620
        if self.value:
            self.value.generate_evaluation_code(code)
5621
            if self.return_type.is_memoryviewslice:
5622
                from . import MemoryView
5623
                MemoryView.put_acquire_memoryviewslice(
5624 5625 5626 5627 5628 5629
                    lhs_cname=Naming.retval_cname,
                    lhs_type=self.return_type,
                    lhs_pos=self.value.pos,
                    rhs=self.value,
                    code=code,
                    have_gil=self.in_nogil_context)
5630 5631
            elif self.in_generator:
                # return value == raise StopIteration(value), but uncatchable
5632
                code.globalstate.use_utility_code(
5633
                    UtilityCode.load_cached("ReturnWithStopIteration", "Coroutine.c"))
5634 5635 5636
                code.putln("%s = NULL; __Pyx_ReturnWithStopIteration(%s);" % (
                    Naming.retval_cname,
                    self.value.py_result()))
5637
                self.value.generate_disposal_code(code)
5638 5639
            else:
                self.value.make_owned_reference(code)
5640 5641 5642
                code.putln("%s = %s;" % (
                    Naming.retval_cname,
                    self.value.result_as(self.return_type)))
5643
            self.value.generate_post_assignment_code(code)
5644
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
5645 5646
        else:
            if self.return_type.is_pyobject:
5647 5648 5649 5650
                if self.in_generator:
                    code.putln("%s = NULL;" % Naming.retval_cname)
                else:
                    code.put_init_to_py_none(Naming.retval_cname, self.return_type)
William Stein's avatar
William Stein committed
5651
            elif self.return_type.is_returncode:
5652 5653
                self.put_return(code, self.return_type.default_value)

5654
        for cname, type in code.funcstate.temps_holding_reference():
5655
            code.put_decref_clear(cname, type)
5656

5657
        code.put_goto(code.return_label)
5658

5659 5660 5661 5662 5663
    def put_return(self, code, value):
        if self.in_parallel:
            code.putln_openmp("#pragma omp critical(__pyx_returning)")
        code.putln("%s = %s;" % (Naming.retval_cname, value))

5664 5665 5666
    def generate_function_definitions(self, env, code):
        if self.value is not None:
            self.value.generate_function_definitions(env, code)
5667

5668 5669 5670
    def annotate(self, code):
        if self.value:
            self.value.annotate(code)
William Stein's avatar
William Stein committed
5671 5672 5673 5674 5675 5676 5677 5678


class RaiseStatNode(StatNode):
    #  raise statement
    #
    #  exc_type    ExprNode or None
    #  exc_value   ExprNode or None
    #  exc_tb      ExprNode or None
Haoyu Bai's avatar
Haoyu Bai committed
5679
    #  cause       ExprNode or None
5680

Haoyu Bai's avatar
Haoyu Bai committed
5681
    child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"]
5682
    is_terminator = True
5683

William Stein's avatar
William Stein committed
5684 5685
    def analyse_expressions(self, env):
        if self.exc_type:
5686 5687
            exc_type = self.exc_type.analyse_types(env)
            self.exc_type = exc_type.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
5688
        if self.exc_value:
5689 5690
            exc_value = self.exc_value.analyse_types(env)
            self.exc_value = exc_value.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
5691
        if self.exc_tb:
5692 5693
            exc_tb = self.exc_tb.analyse_types(env)
            self.exc_tb = exc_tb.coerce_to_pyobject(env)
Haoyu Bai's avatar
Haoyu Bai committed
5694
        if self.cause:
5695 5696
            cause = self.cause.analyse_types(env)
            self.cause = cause.coerce_to_pyobject(env)
5697 5698 5699 5700
        # special cases for builtin exceptions
        self.builtin_exc_name = None
        if self.exc_type and not self.exc_value and not self.exc_tb:
            exc = self.exc_type
5701
            from . import ExprNodes
Robert Bradshaw's avatar
Robert Bradshaw committed
5702
            if (isinstance(exc, ExprNodes.SimpleCallNode) and
5703 5704
                    not (exc.args or (exc.arg_tuple is not None and exc.arg_tuple.args))):
                exc = exc.function  # extract the exception type
5705 5706 5707 5708
            if exc.is_name and exc.entry.is_builtin:
                self.builtin_exc_name = exc.name
                if self.builtin_exc_name == 'MemoryError':
                    self.exc_type = None # has a separate implementation
5709
        return self
5710

5711
    nogil_check = Node.gil_error
5712 5713
    gil_message = "Raising exception"

William Stein's avatar
William Stein committed
5714
    def generate_execution_code(self, code):
5715
        code.mark_pos(self.pos)
5716 5717 5718 5719
        if self.builtin_exc_name == 'MemoryError':
            code.putln('PyErr_NoMemory(); %s' % code.error_goto(self.pos))
            return

William Stein's avatar
William Stein committed
5720 5721 5722
        if self.exc_type:
            self.exc_type.generate_evaluation_code(code)
            type_code = self.exc_type.py_result()
5723 5724
            if self.exc_type.is_name:
                code.globalstate.use_entry_utility_code(self.exc_type.entry)
William Stein's avatar
William Stein committed
5725
        else:
Stefan Behnel's avatar
Stefan Behnel committed
5726
            type_code = "0"
William Stein's avatar
William Stein committed
5727 5728 5729 5730 5731 5732 5733 5734 5735 5736
        if self.exc_value:
            self.exc_value.generate_evaluation_code(code)
            value_code = self.exc_value.py_result()
        else:
            value_code = "0"
        if self.exc_tb:
            self.exc_tb.generate_evaluation_code(code)
            tb_code = self.exc_tb.py_result()
        else:
            tb_code = "0"
Haoyu Bai's avatar
Haoyu Bai committed
5737 5738 5739 5740 5741
        if self.cause:
            self.cause.generate_evaluation_code(code)
            cause_code = self.cause.py_result()
        else:
            cause_code = "0"
5742
        code.globalstate.use_utility_code(raise_utility_code)
5743
        code.putln(
Haoyu Bai's avatar
Haoyu Bai committed
5744
            "__Pyx_Raise(%s, %s, %s, %s);" % (
5745 5746
                type_code,
                value_code,
Haoyu Bai's avatar
Haoyu Bai committed
5747 5748 5749
                tb_code,
                cause_code))
        for obj in (self.exc_type, self.exc_value, self.exc_tb, self.cause):
5750 5751 5752
            if obj:
                obj.generate_disposal_code(code)
                obj.free_temps(code)
William Stein's avatar
William Stein committed
5753 5754 5755
        code.putln(
            code.error_goto(self.pos))

5756 5757 5758 5759 5760 5761 5762
    def generate_function_definitions(self, env, code):
        if self.exc_type is not None:
            self.exc_type.generate_function_definitions(env, code)
        if self.exc_value is not None:
            self.exc_value.generate_function_definitions(env, code)
        if self.exc_tb is not None:
            self.exc_tb.generate_function_definitions(env, code)
Haoyu Bai's avatar
Haoyu Bai committed
5763 5764
        if self.cause is not None:
            self.cause.generate_function_definitions(env, code)
5765

5766 5767 5768 5769 5770 5771 5772
    def annotate(self, code):
        if self.exc_type:
            self.exc_type.annotate(code)
        if self.exc_value:
            self.exc_value.annotate(code)
        if self.exc_tb:
            self.exc_tb.annotate(code)
Haoyu Bai's avatar
Haoyu Bai committed
5773 5774
        if self.cause:
            self.cause.annotate(code)
5775

William Stein's avatar
William Stein committed
5776

5777 5778
class ReraiseStatNode(StatNode):

5779
    child_attrs = []
5780
    is_terminator = True
5781

5782
    def analyse_expressions(self, env):
5783
        return self
5784

5785
    nogil_check = Node.gil_error
5786 5787
    gil_message = "Raising exception"

5788
    def generate_execution_code(self, code):
5789
        code.mark_pos(self.pos)
5790
        vars = code.funcstate.exc_vars
5791
        if vars:
5792
            code.globalstate.use_utility_code(restore_exception_utility_code)
5793 5794 5795 5796
            code.put_giveref(vars[0])
            code.put_giveref(vars[1])
            # fresh exceptions may not have a traceback yet (-> finally!)
            code.put_xgiveref(vars[2])
5797
            code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars))
5798 5799 5800
            for varname in vars:
                code.put("%s = 0; " % varname)
            code.putln()
5801 5802
            code.putln(code.error_goto(self.pos))
        else:
5803 5804 5805
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ReRaiseException", "Exceptions.c"))
            code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos))
5806

William Stein's avatar
William Stein committed
5807 5808 5809 5810 5811
class AssertStatNode(StatNode):
    #  assert statement
    #
    #  cond    ExprNode
    #  value   ExprNode or None
5812

5813 5814
    child_attrs = ["cond", "value"]

William Stein's avatar
William Stein committed
5815 5816 5817
    def analyse_expressions(self, env):
        self.cond = self.cond.analyse_boolean_expression(env)
        if self.value:
5818
            value = self.value.analyse_types(env)
5819 5820
            if value.type is Builtin.tuple_type or not value.type.is_builtin_type:
                # prevent tuple values from being interpreted as argument value tuples
5821
                from .ExprNodes import TupleNode
5822
                value = TupleNode(value.pos, args=[value], slow=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
5823
                self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env)
5824 5825
            else:
                self.value = value.coerce_to_pyobject(env)
5826
        return self
5827

5828
    nogil_check = Node.gil_error
5829
    gil_message = "Raising exception"
5830

William Stein's avatar
William Stein committed
5831
    def generate_execution_code(self, code):
5832
        code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS")
5833
        code.putln("if (unlikely(!Py_OptimizeFlag)) {")
5834
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5835 5836
        self.cond.generate_evaluation_code(code)
        code.putln(
5837
            "if (unlikely(!%s)) {" % self.cond.result())
William Stein's avatar
William Stein committed
5838
        if self.value:
5839
            self.value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
5840
            code.putln(
5841
                "PyErr_SetObject(PyExc_AssertionError, %s);" % self.value.py_result())
5842
            self.value.generate_disposal_code(code)
5843
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
5844 5845 5846 5847
        else:
            code.putln(
                "PyErr_SetNone(PyExc_AssertionError);")
        code.putln(
5848
            code.error_goto(self.pos))
William Stein's avatar
William Stein committed
5849 5850 5851
        code.putln(
            "}")
        self.cond.generate_disposal_code(code)
5852
        self.cond.free_temps(code)
5853 5854
        code.putln(
            "}")
5855
        code.putln("#endif")
William Stein's avatar
William Stein committed
5856

5857 5858 5859 5860 5861
    def generate_function_definitions(self, env, code):
        self.cond.generate_function_definitions(env, code)
        if self.value is not None:
            self.value.generate_function_definitions(env, code)

5862 5863 5864 5865 5866 5867
    def annotate(self, code):
        self.cond.annotate(code)
        if self.value:
            self.value.annotate(code)


William Stein's avatar
William Stein committed
5868 5869 5870 5871 5872
class IfStatNode(StatNode):
    #  if statement
    #
    #  if_clauses   [IfClauseNode]
    #  else_clause  StatNode or None
5873 5874

    child_attrs = ["if_clauses", "else_clause"]
5875

William Stein's avatar
William Stein committed
5876 5877 5878 5879 5880
    def analyse_declarations(self, env):
        for if_clause in self.if_clauses:
            if_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
5881

William Stein's avatar
William Stein committed
5882
    def analyse_expressions(self, env):
5883
        self.if_clauses = [if_clause.analyse_expressions(env) for if_clause in self.if_clauses]
William Stein's avatar
William Stein committed
5884
        if self.else_clause:
5885 5886
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
5887

William Stein's avatar
William Stein committed
5888
    def generate_execution_code(self, code):
5889
        code.mark_pos(self.pos)
5890
        end_label = code.new_label()
5891 5892 5893
        last = len(self.if_clauses)
        if not self.else_clause:
            last -= 1  # avoid redundant goto at end of last if-clause
5894 5895
        for i, if_clause in enumerate(self.if_clauses):
            if_clause.generate_execution_code(code, end_label, is_last=i == last)
5896
        if self.else_clause:
5897
            code.mark_pos(self.else_clause.pos)
5898
            code.putln("/*else*/ {")
William Stein's avatar
William Stein committed
5899
            self.else_clause.generate_execution_code(code)
5900 5901
            code.putln("}")
        code.put_label(end_label)
5902

5903 5904 5905 5906 5907
    def generate_function_definitions(self, env, code):
        for clause in self.if_clauses:
            clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
5908

5909 5910 5911 5912 5913
    def annotate(self, code):
        for if_clause in self.if_clauses:
            if_clause.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
5914 5915 5916 5917 5918 5919 5920


class IfClauseNode(Node):
    #  if or elif clause in an if statement
    #
    #  condition   ExprNode
    #  body        StatNode
5921

5922 5923
    child_attrs = ["condition", "body"]

William Stein's avatar
William Stein committed
5924 5925
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
5926

William Stein's avatar
William Stein committed
5927
    def analyse_expressions(self, env):
5928
        self.condition = self.condition.analyse_temp_boolean_expression(env)
5929 5930
        self.body = self.body.analyse_expressions(env)
        return self
5931

5932
    def generate_execution_code(self, code, end_label, is_last):
William Stein's avatar
William Stein committed
5933
        self.condition.generate_evaluation_code(code)
5934 5935
        code.mark_pos(self.pos)
        code.putln("if (%s) {" % self.condition.result())
5936 5937
        self.condition.generate_disposal_code(code)
        self.condition.free_temps(code)
William Stein's avatar
William Stein committed
5938
        self.body.generate_execution_code(code)
5939 5940
        code.mark_pos(self.pos, trace=False)
        if not (is_last or self.body.is_terminator):
5941
            code.put_goto(end_label)
William Stein's avatar
William Stein committed
5942
        code.putln("}")
5943

5944 5945 5946 5947
    def generate_function_definitions(self, env, code):
        self.condition.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)

5948 5949 5950
    def annotate(self, code):
        self.condition.annotate(code)
        self.body.annotate(code)
5951

5952 5953 5954 5955 5956 5957

class SwitchCaseNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # conditions    [ExprNode]
    # body          StatNode
5958

5959
    child_attrs = ['conditions', 'body']
5960

5961 5962
    def generate_execution_code(self, code):
        for cond in self.conditions:
5963
            code.mark_pos(cond.pos)
5964 5965
            cond.generate_evaluation_code(code)
            code.putln("case %s:" % cond.result())
5966
        self.body.generate_execution_code(code)
5967
        code.mark_pos(self.pos, trace=False)
5968
        code.putln("break;")
5969 5970 5971 5972 5973

    def generate_function_definitions(self, env, code):
        for cond in self.conditions:
            cond.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
5974

5975 5976 5977
    def annotate(self, code):
        for cond in self.conditions:
            cond.annotate(code)
5978
        self.body.annotate(code)
5979

5980

5981 5982 5983 5984 5985 5986
class SwitchStatNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # test          ExprNode
    # cases         [SwitchCaseNode]
    # else_clause   StatNode or None
5987

5988
    child_attrs = ['test', 'cases', 'else_clause']
5989

5990
    def generate_execution_code(self, code):
5991
        self.test.generate_evaluation_code(code)
5992
        code.mark_pos(self.pos)
5993
        code.putln("switch (%s) {" % self.test.result())
5994 5995 5996 5997 5998
        for case in self.cases:
            case.generate_execution_code(code)
        if self.else_clause is not None:
            code.putln("default:")
            self.else_clause.generate_execution_code(code)
5999
            code.putln("break;")
6000 6001 6002 6003 6004
        else:
            # Always generate a default clause to prevent C compiler warnings
            # about unmatched enum values (it was not the user who decided to
            # generate the switch statement, so shouldn't be bothered).
            code.putln("default: break;")
6005 6006
        code.putln("}")

6007 6008 6009 6010 6011 6012 6013
    def generate_function_definitions(self, env, code):
        self.test.generate_function_definitions(env, code)
        for case in self.cases:
            case.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6014 6015 6016 6017
    def annotate(self, code):
        self.test.annotate(code)
        for case in self.cases:
            case.annotate(code)
6018 6019
        if self.else_clause is not None:
            self.else_clause.annotate(code)
6020

6021

6022
class LoopNode(object):
6023
    pass
6024

6025

6026
class WhileStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6027 6028 6029 6030 6031
    #  while statement
    #
    #  condition    ExprNode
    #  body         StatNode
    #  else_clause  StatNode
6032 6033

    child_attrs = ["condition", "body", "else_clause"]
6034

William Stein's avatar
William Stein committed
6035 6036 6037 6038
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6039

William Stein's avatar
William Stein committed
6040
    def analyse_expressions(self, env):
6041 6042
        if self.condition:
            self.condition = self.condition.analyse_temp_boolean_expression(env)
6043
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6044
        if self.else_clause:
6045 6046
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6047

William Stein's avatar
William Stein committed
6048
    def generate_execution_code(self, code):
6049
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6050 6051 6052
        old_loop_labels = code.new_loop_labels()
        code.putln(
            "while (1) {")
6053 6054 6055 6056
        if self.condition:
            self.condition.generate_evaluation_code(code)
            self.condition.generate_disposal_code(code)
            code.putln(
6057
                "if (!%s) break;" % self.condition.result())
6058
            self.condition.free_temps(code)
William Stein's avatar
William Stein committed
6059
        self.body.generate_execution_code(code)
6060
        code.put_label(code.continue_label)
William Stein's avatar
William Stein committed
6061 6062 6063 6064
        code.putln("}")
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
6065
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
6066 6067 6068 6069 6070
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)

6071
    def generate_function_definitions(self, env, code):
6072 6073
        if self.condition:
            self.condition.generate_function_definitions(env, code)
6074 6075 6076 6077
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6078
    def annotate(self, code):
6079 6080
        if self.condition:
            self.condition.annotate(code)
6081 6082 6083 6084
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
6085

6086 6087 6088 6089
class DictIterationNextNode(Node):
    # Helper node for calling PyDict_Next() inside of a WhileStatNode
    # and checking the dictionary size for changes.  Created in
    # Optimize.py.
6090 6091 6092
    child_attrs = ['dict_obj', 'expected_size', 'pos_index_var',
                   'coerced_key_var', 'coerced_value_var', 'coerced_tuple_var',
                   'key_target', 'value_target', 'tuple_target', 'is_dict_flag']
6093

6094 6095 6096 6097 6098 6099
    coerced_key_var = key_ref = None
    coerced_value_var = value_ref = None
    coerced_tuple_var = tuple_ref = None

    def __init__(self, dict_obj, expected_size, pos_index_var,
                 key_target, value_target, tuple_target, is_dict_flag):
6100 6101
        Node.__init__(
            self, dict_obj.pos,
6102 6103 6104 6105 6106 6107 6108 6109 6110
            dict_obj=dict_obj,
            expected_size=expected_size,
            pos_index_var=pos_index_var,
            key_target=key_target,
            value_target=value_target,
            tuple_target=tuple_target,
            is_dict_flag=is_dict_flag,
            is_temp=True,
            type=PyrexTypes.c_bint_type)
6111 6112

    def analyse_expressions(self, env):
6113
        from . import ExprNodes
6114 6115 6116 6117
        self.dict_obj = self.dict_obj.analyse_types(env)
        self.expected_size = self.expected_size.analyse_types(env)
        if self.pos_index_var:
            self.pos_index_var = self.pos_index_var.analyse_types(env)
6118
        if self.key_target:
6119
            self.key_target = self.key_target.analyse_target_types(env)
6120
            self.key_ref = ExprNodes.TempNode(self.key_target.pos, PyrexTypes.py_object_type)
6121 6122
            self.coerced_key_var = self.key_ref.coerce_to(self.key_target.type, env)
        if self.value_target:
6123
            self.value_target = self.value_target.analyse_target_types(env)
6124
            self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
6125 6126
            self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
        if self.tuple_target:
6127
            self.tuple_target = self.tuple_target.analyse_target_types(env)
6128
            self.tuple_ref = ExprNodes.TempNode(self.tuple_target.pos, PyrexTypes.py_object_type)
6129
            self.coerced_tuple_var = self.tuple_ref.coerce_to(self.tuple_target.type, env)
6130 6131
        self.is_dict_flag = self.is_dict_flag.analyse_types(env)
        return self
6132 6133 6134 6135 6136

    def generate_function_definitions(self, env, code):
        self.dict_obj.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
6137
        code.globalstate.use_utility_code(UtilityCode.load_cached("dict_iter", "Optimize.c"))
6138 6139
        self.dict_obj.generate_evaluation_code(code)

6140 6141 6142 6143 6144 6145 6146 6147
        assignments = []
        temp_addresses = []
        for var, result, target in [(self.key_ref, self.coerced_key_var, self.key_target),
                                    (self.value_ref, self.coerced_value_var, self.value_target),
                                    (self.tuple_ref, self.coerced_tuple_var, self.tuple_target)]:
            if target is None:
                addr = 'NULL'
            else:
6148 6149 6150
                assignments.append((var, result, target))
                var.allocate(code)
                addr = '&%s' % var.result()
6151 6152 6153 6154 6155
            temp_addresses.append(addr)

        result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
        code.putln("%s = __Pyx_dict_iter_next(%s, %s, &%s, %s, %s, %s, %s);" % (
            result_temp,
6156
            self.dict_obj.py_result(),
6157 6158 6159 6160 6161 6162 6163 6164 6165 6166 6167 6168
            self.expected_size.result(),
            self.pos_index_var.result(),
            temp_addresses[0],
            temp_addresses[1],
            temp_addresses[2],
            self.is_dict_flag.result()
        ))
        code.putln("if (unlikely(%s == 0)) break;" % result_temp)
        code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
        code.funcstate.release_temp(result_temp)

        # evaluate all coercions before the assignments
6169 6170
        for var, result, target in assignments:
            code.put_gotref(var.result())
6171
        for var, result, target in assignments:
6172
            result.generate_evaluation_code(code)
6173
        for var, result, target in assignments:
6174
            target.generate_assignment_code(result, code)
6175
            var.release(code)
6176

6177

Robert Bradshaw's avatar
Robert Bradshaw committed
6178
def ForStatNode(pos, **kw):
6179
    if 'iterator' in kw:
6180 6181 6182 6183
        if kw['iterator'].is_async:
            return AsyncForStatNode(pos, **kw)
        else:
            return ForInStatNode(pos, **kw)
Robert Bradshaw's avatar
Robert Bradshaw committed
6184 6185 6186
    else:
        return ForFromStatNode(pos, **kw)

6187 6188 6189

class _ForInStatNode(LoopNode, StatNode):
    #  Base class of 'for-in' statements.
William Stein's avatar
William Stein committed
6190 6191
    #
    #  target        ExprNode
6192
    #  iterator      IteratorNode | AIterAwaitExprNode(AsyncIteratorNode)
William Stein's avatar
William Stein committed
6193 6194
    #  body          StatNode
    #  else_clause   StatNode
6195 6196
    #  item          NextNode | AwaitExprNode(AsyncNextNode)
    #  is_async      boolean        true for 'async for' statements
6197

6198
    child_attrs = ["target", "item", "iterator", "body", "else_clause"]
6199
    item = None
6200 6201 6202 6203
    is_async = False

    def _create_item_node(self):
        raise NotImplementedError("must be implemented by subclasses")
6204

William Stein's avatar
William Stein committed
6205 6206 6207 6208 6209
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6210
        self._create_item_node()
6211

William Stein's avatar
William Stein committed
6212
    def analyse_expressions(self, env):
6213
        self.target = self.target.analyse_target_types(env)
6214
        self.iterator = self.iterator.analyse_expressions(env)
6215
        self._create_item_node()  # must rewrap self.item after analysis
6216
        self.item = self.item.analyse_expressions(env)
6217 6218 6219
        if (not self.is_async and
                (self.iterator.type.is_ptr or self.iterator.type.is_array) and
                self.target.type.assignable_from(self.iterator.type)):
6220 6221 6222 6223
            # C array slice optimization.
            pass
        else:
            self.item = self.item.coerce_to(self.target.type, env)
6224
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6225
        if self.else_clause:
6226 6227
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
William Stein's avatar
William Stein committed
6228 6229

    def generate_execution_code(self, code):
6230
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6231 6232
        old_loop_labels = code.new_loop_labels()
        self.iterator.generate_evaluation_code(code)
Mark Florisson's avatar
Mark Florisson committed
6233
        code.putln("for (;;) {")
William Stein's avatar
William Stein committed
6234 6235 6236
        self.item.generate_evaluation_code(code)
        self.target.generate_assignment_code(self.item, code)
        self.body.generate_execution_code(code)
6237
        code.mark_pos(self.pos)
6238
        code.put_label(code.continue_label)
Mark Florisson's avatar
Mark Florisson committed
6239
        code.putln("}")
William Stein's avatar
William Stein committed
6240 6241
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
6242

William Stein's avatar
William Stein committed
6243
        if self.else_clause:
6244 6245 6246 6247 6248 6249 6250
            # in nested loops, the 'else' block can contain a
            # 'continue' statement for the outer loop, but we may need
            # to generate cleanup code before taking that path, so we
            # intercept it here
            orig_continue_label = code.continue_label
            code.continue_label = code.new_label('outer_continue')

William Stein's avatar
William Stein committed
6251 6252 6253
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
6254 6255 6256

            if code.label_used(code.continue_label):
                code.put_goto(break_label)
6257
                code.mark_pos(self.pos)
6258 6259 6260 6261 6262
                code.put_label(code.continue_label)
                self.iterator.generate_disposal_code(code)
                code.put_goto(orig_continue_label)
            code.set_loop_labels(old_loop_labels)

6263
        code.mark_pos(self.pos)
6264 6265
        if code.label_used(break_label):
            code.put_label(break_label)
William Stein's avatar
William Stein committed
6266
        self.iterator.generate_disposal_code(code)
6267
        self.iterator.free_temps(code)
William Stein's avatar
William Stein committed
6268

6269 6270 6271 6272 6273 6274 6275
    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.iterator.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6276 6277 6278 6279 6280 6281 6282 6283
    def annotate(self, code):
        self.target.annotate(code)
        self.iterator.annotate(code)
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
        self.item.annotate(code)

William Stein's avatar
William Stein committed
6284

6285 6286 6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297
class ForInStatNode(_ForInStatNode):
    #  'for' statement

    is_async = False

    def _create_item_node(self):
        from .ExprNodes import NextNode
        self.item = NextNode(self.iterator)


class AsyncForStatNode(_ForInStatNode):
    #  'async for' statement
    #
6298
    #  iterator      AIterAwaitExprNode(AsyncIteratorNode)
6299 6300 6301 6302 6303 6304 6305 6306
    #  item          AwaitIterNextExprNode(AsyncIteratorNode)

    is_async = True

    def __init__(self, pos, iterator, **kw):
        assert 'item' not in kw
        from . import ExprNodes
        # AwaitExprNodes must appear before running MarkClosureVisitor
6307
        kw['iterator'] = ExprNodes.AIterAwaitExprNode(iterator.pos, arg=iterator)
6308 6309 6310 6311 6312 6313 6314 6315
        kw['item'] = ExprNodes.AwaitIterNextExprNode(iterator.pos, arg=None)
        _ForInStatNode.__init__(self, pos, **kw)

    def _create_item_node(self):
        from . import ExprNodes
        self.item.arg = ExprNodes.AsyncNextNode(self.iterator)


6316
class ForFromStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6317 6318 6319 6320 6321 6322 6323
    #  for name from expr rel name rel expr
    #
    #  target        NameNode
    #  bound1        ExprNode
    #  relation1     string
    #  relation2     string
    #  bound2        ExprNode
6324
    #  step          ExprNode or None
William Stein's avatar
William Stein committed
6325 6326 6327 6328 6329
    #  body          StatNode
    #  else_clause   StatNode or None
    #
    #  Used internally:
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
6330
    #  from_range         bool
6331
    #  is_py_target       bool
6332
    #  loopvar_node       ExprNode (usually a NameNode or temp node)
William Stein's avatar
William Stein committed
6333
    #  py_loopvar_node    PyTempNode or None
6334
    child_attrs = ["target", "bound1", "bound2", "step", "body", "else_clause"]
6335 6336

    is_py_target = False
6337
    loopvar_node = None
6338
    py_loopvar_node = None
Robert Bradshaw's avatar
Robert Bradshaw committed
6339
    from_range = False
6340

6341 6342 6343 6344 6345 6346 6347
    gil_message = "For-loop using object bounds or target"

    def nogil_check(self, env):
        for x in (self.target, self.bound1, self.bound2):
            if x.type.is_pyobject:
                self.gil_error()

Robert Bradshaw's avatar
Robert Bradshaw committed
6348 6349 6350 6351 6352
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6353

William Stein's avatar
William Stein committed
6354
    def analyse_expressions(self, env):
6355
        from . import ExprNodes
6356 6357 6358
        self.target = self.target.analyse_target_types(env)
        self.bound1 = self.bound1.analyse_types(env)
        self.bound2 = self.bound2.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6359 6360
        if self.step is not None:
            if isinstance(self.step, ExprNodes.UnaryMinusNode):
6361 6362
                warning(self.step.pos, "Probable infinite loop in for-from-by statement. "
                        "Consider switching the directions of the relations.", 2)
6363
            self.step = self.step.analyse_types(env)
6364

6365
        if self.target.type.is_numeric:
Robert Bradshaw's avatar
Robert Bradshaw committed
6366
            loop_type = self.target.type
6367
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6368 6369 6370 6371 6372 6373 6374 6375 6376
            loop_type = PyrexTypes.c_int_type
            if not self.bound1.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound1.type)
            if not self.bound2.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound2.type)
            if self.step is not None and not self.step.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.step.type)
        self.bound1 = self.bound1.coerce_to(loop_type, env)
        self.bound2 = self.bound2.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6377 6378
        if not self.bound2.is_literal:
            self.bound2 = self.bound2.coerce_to_temp(env)
6379
        if self.step is not None:
6380
            self.step = self.step.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6381 6382
            if not self.step.is_literal:
                self.step = self.step.coerce_to_temp(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6383

William Stein's avatar
William Stein committed
6384
        target_type = self.target.type
6385
        if not (target_type.is_pyobject or target_type.is_numeric):
6386
            error(self.target.pos, "for-from loop variable must be c numeric type or Python object")
6387
        if target_type.is_numeric:
Robert Bradshaw's avatar
Robert Bradshaw committed
6388
            self.is_py_target = False
6389 6390
            if isinstance(self.target, ExprNodes.BufferIndexNode):
                raise error(self.pos, "Buffer or memoryview slicing/indexing not allowed as for-loop target.")
6391
            self.loopvar_node = self.target
William Stein's avatar
William Stein committed
6392 6393
            self.py_loopvar_node = None
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6394 6395
            self.is_py_target = True
            c_loopvar_node = ExprNodes.TempNode(self.pos, loop_type, env)
6396
            self.loopvar_node = c_loopvar_node
William Stein's avatar
William Stein committed
6397 6398
            self.py_loopvar_node = \
                ExprNodes.CloneNode(c_loopvar_node).coerce_to_pyobject(env)
6399
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6400
        if self.else_clause:
6401 6402
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6403

William Stein's avatar
William Stein committed
6404
    def generate_execution_code(self, code):
6405
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6406
        old_loop_labels = code.new_loop_labels()
Robert Bradshaw's avatar
Robert Bradshaw committed
6407
        from_range = self.from_range
William Stein's avatar
William Stein committed
6408 6409 6410
        self.bound1.generate_evaluation_code(code)
        self.bound2.generate_evaluation_code(code)
        offset, incop = self.relation_table[self.relation1]
6411 6412
        if self.step is not None:
            self.step.generate_evaluation_code(code)
Magnus Lie Hetland's avatar
Magnus Lie Hetland committed
6413 6414
            step = self.step.result()
            incop = "%s=%s" % (incop[0], step)
6415
        from . import ExprNodes
6416 6417 6418 6419
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.allocate(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.allocate(code)
6420
        if from_range:
Robert Bradshaw's avatar
Robert Bradshaw committed
6421
            loopvar_name = code.funcstate.allocate_temp(self.target.type, False)
6422
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6423
            loopvar_name = self.loopvar_node.result()
6424 6425 6426 6427 6428
        if self.target.type.is_int and not self.target.type.signed and self.relation2[0] == '>':
            # Handle the case where the endpoint of an unsigned int iteration
            # is within step of 0.
            if not self.step:
                step = 1
6429 6430 6431 6432 6433
            code.putln("for (%s = %s%s + %s; %s %s %s + %s; ) { %s%s;" % (
                loopvar_name,
                self.bound1.result(), offset, step,
                loopvar_name, self.relation2, self.bound2.result(), step,
                loopvar_name, incop))
6434
        else:
6435 6436 6437 6438 6439
            code.putln("for (%s = %s%s; %s %s %s; %s%s) {" % (
                loopvar_name,
                self.bound1.result(), offset,
                loopvar_name, self.relation2, self.bound2.result(),
                loopvar_name, incop))
William Stein's avatar
William Stein committed
6440 6441 6442
        if self.py_loopvar_node:
            self.py_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(self.py_loopvar_node, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
6443 6444
        elif from_range:
            code.putln("%s = %s;" % (
6445
                self.target.result(), loopvar_name))
William Stein's avatar
William Stein committed
6446 6447
        self.body.generate_execution_code(code)
        code.put_label(code.continue_label)
Robert Bradshaw's avatar
Robert Bradshaw committed
6448
        if self.py_loopvar_node:
6449 6450 6451
            # This mess is to make for..from loops with python targets behave
            # exactly like those with C targets with regards to re-assignment
            # of the loop variable.
6452
            if self.target.entry.is_pyglobal:
6453
                # We know target is a NameNode, this is the only ugly case.
6454
                target_node = ExprNodes.PyTempNode(self.target.pos, None)
6455 6456
                target_node.allocate(code)
                interned_cname = code.intern_identifier(self.target.entry.name)
6457 6458 6459 6460 6461 6462 6463 6464 6465 6466
                if self.target.entry.scope.is_module_scope:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c"))
                    lookup_func = '__Pyx_GetModuleGlobalName(%s)'
                else:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetNameInClass", "ObjectHandling.c"))
                    lookup_func = '__Pyx_GetNameInClass(%s, %%s)' % (
                        self.target.entry.scope.namespace_cname)
                code.putln("%s = %s; %s" % (
6467
                    target_node.result(),
6468
                    lookup_func % interned_cname,
6469
                    code.error_goto_if_null(target_node.result(), self.target.pos)))
6470
                code.put_gotref(target_node.result())
6471 6472
            else:
                target_node = self.target
6473 6474
            from_py_node = ExprNodes.CoerceFromPyTypeNode(
                self.loopvar_node.type, target_node, self.target.entry.scope)
6475 6476
            from_py_node.temp_code = loopvar_name
            from_py_node.generate_result_code(code)
6477
            if self.target.entry.is_pyglobal:
6478 6479
                code.put_decref(target_node.result(), target_node.type)
                target_node.release(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
6480
        code.putln("}")
6481
        if self.py_loopvar_node:
6482 6483
            # This is potentially wasteful, but we don't want the semantics to
            # depend on whether or not the loop is a python type.
6484 6485
            self.py_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(self.py_loopvar_node, code)
6486 6487
        if from_range:
            code.funcstate.release_temp(loopvar_name)
William Stein's avatar
William Stein committed
6488 6489 6490 6491 6492 6493 6494 6495
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)
        self.bound1.generate_disposal_code(code)
6496
        self.bound1.free_temps(code)
William Stein's avatar
William Stein committed
6497
        self.bound2.generate_disposal_code(code)
6498
        self.bound2.free_temps(code)
6499 6500 6501 6502
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.release(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.release(code)
6503 6504
        if self.step is not None:
            self.step.generate_disposal_code(code)
6505
            self.step.free_temps(code)
6506

William Stein's avatar
William Stein committed
6507 6508 6509 6510 6511
    relation_table = {
        # {relop : (initial offset, increment op)}
        '<=': ("",   "++"),
        '<' : ("+1", "++"),
        '>=': ("",   "--"),
6512
        '>' : ("-1", "--"),
William Stein's avatar
William Stein committed
6513
    }
6514 6515 6516 6517 6518 6519 6520 6521 6522 6523

    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.bound1.generate_function_definitions(env, code)
        self.bound2.generate_function_definitions(env, code)
        if self.step is not None:
            self.step.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
6524

6525 6526 6527 6528 6529
    def annotate(self, code):
        self.target.annotate(code)
        self.bound1.annotate(code)
        self.bound2.annotate(code)
        if self.step:
6530
            self.step.annotate(code)
6531 6532 6533
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
6534 6535


6536 6537 6538
class WithStatNode(StatNode):
    """
    Represents a Python with statement.
6539

6540
    Implemented by the WithTransform as follows:
6541 6542 6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557

        MGR = EXPR
        EXIT = MGR.__exit__
        VALUE = MGR.__enter__()
        EXC = True
        try:
            try:
                TARGET = VALUE  # optional
                BODY
            except:
                EXC = False
                if not EXIT(*EXCINFO):
                    raise
        finally:
            if EXC:
                EXIT(None, None, None)
            MGR = EXIT = VALUE = None
6558 6559
    """
    #  manager          The with statement manager object
6560
    #  target           ExprNode  the target lhs of the __enter__() call
6561
    #  body             StatNode
6562
    #  enter_call       ExprNode  the call to the __enter__() method
6563
    #  exit_var         String    the cname of the __exit__() method reference
6564

6565
    child_attrs = ["manager", "enter_call", "target", "body"]
6566

6567
    enter_call = None
6568
    target_temp = None
6569 6570 6571

    def analyse_declarations(self, env):
        self.manager.analyse_declarations(env)
6572
        self.enter_call.analyse_declarations(env)
6573 6574 6575
        self.body.analyse_declarations(env)

    def analyse_expressions(self, env):
6576 6577
        self.manager = self.manager.analyse_types(env)
        self.enter_call = self.enter_call.analyse_types(env)
6578 6579 6580 6581
        if self.target:
            # set up target_temp before descending into body (which uses it)
            from .ExprNodes import TempNode
            self.target_temp = TempNode(self.enter_call.pos, self.enter_call.type)
6582 6583
        self.body = self.body.analyse_expressions(env)
        return self
6584

6585 6586
    def generate_function_definitions(self, env, code):
        self.manager.generate_function_definitions(env, code)
6587
        self.enter_call.generate_function_definitions(env, code)
6588 6589
        self.body.generate_function_definitions(env, code)

6590
    def generate_execution_code(self, code):
6591
        code.mark_pos(self.pos)
6592 6593 6594
        code.putln("/*with:*/ {")
        self.manager.generate_evaluation_code(code)
        self.exit_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
6595
        code.globalstate.use_utility_code(
6596 6597
            UtilityCode.load_cached("PyObjectLookupSpecial", "ObjectHandling.c"))
        code.putln("%s = __Pyx_PyObject_LookupSpecial(%s, %s); %s" % (
6598 6599
            self.exit_var,
            self.manager.py_result(),
6600
            code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')),
6601 6602 6603 6604 6605 6606 6607 6608
            code.error_goto_if_null(self.exit_var, self.pos),
            ))
        code.put_gotref(self.exit_var)

        # need to free exit_var in the face of exceptions during setup
        old_error_label = code.new_error_label()
        intermediate_error_label = code.error_label

6609
        self.enter_call.generate_evaluation_code(code)
6610 6611 6612 6613 6614 6615 6616
        if self.target:
            # The temp result will be cleaned up by the WithTargetAssignmentStatNode
            # after assigning its result to the target of the 'with' statement.
            self.target_temp.allocate(code)
            self.enter_call.make_owned_reference(code)
            code.putln("%s = %s;" % (self.target_temp.result(), self.enter_call.result()))
            self.enter_call.generate_post_assignment_code(code)
6617
        else:
6618 6619 6620
            self.enter_call.generate_disposal_code(code)
        self.enter_call.free_temps(code)

6621 6622 6623 6624 6625 6626
        self.manager.generate_disposal_code(code)
        self.manager.free_temps(code)

        code.error_label = old_error_label
        self.body.generate_execution_code(code)

6627 6628 6629 6630 6631 6632 6633
        if code.label_used(intermediate_error_label):
            step_over_label = code.new_label()
            code.put_goto(step_over_label)
            code.put_label(intermediate_error_label)
            code.put_decref_clear(self.exit_var, py_object_type)
            code.put_goto(old_error_label)
            code.put_label(step_over_label)
6634 6635 6636 6637

        code.funcstate.release_temp(self.exit_var)
        code.putln('}')

6638

6639 6640 6641 6642
class WithTargetAssignmentStatNode(AssignmentNode):
    # The target assignment of the 'with' statement value (return
    # value of the __enter__() call).
    #
6643
    # This is a special cased assignment that properly cleans up the RHS.
6644
    #
6645 6646 6647
    # lhs       ExprNode      the assignment target
    # rhs       ExprNode      a (coerced) TempNode for the rhs (from WithStatNode)
    # with_node WithStatNode  the surrounding with-statement
6648

6649 6650 6651
    child_attrs = ["rhs", "lhs"]
    with_node = None
    rhs = None
6652 6653 6654 6655

    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)

6656
    def analyse_expressions(self, env):
6657
        self.lhs = self.lhs.analyse_target_types(env)
6658
        self.lhs.gil_assignment_check(env)
6659
        self.rhs = self.with_node.target_temp.coerce_to(self.lhs.type, env)
6660
        return self
6661 6662 6663 6664

    def generate_execution_code(self, code):
        self.rhs.generate_evaluation_code(code)
        self.lhs.generate_assignment_code(self.rhs, code)
6665
        self.with_node.target_temp.release(code)
6666 6667 6668 6669 6670

    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

6671

William Stein's avatar
William Stein committed
6672 6673 6674 6675 6676 6677
class TryExceptStatNode(StatNode):
    #  try .. except statement
    #
    #  body             StatNode
    #  except_clauses   [ExceptClauseNode]
    #  else_clause      StatNode or None
6678

6679
    child_attrs = ["body", "except_clauses", "else_clause"]
6680

William Stein's avatar
William Stein committed
6681 6682 6683 6684 6685 6686
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        for except_clause in self.except_clauses:
            except_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6687

William Stein's avatar
William Stein committed
6688
    def analyse_expressions(self, env):
6689
        self.body = self.body.analyse_expressions(env)
6690
        default_clause_seen = 0
6691 6692
        for i, except_clause in enumerate(self.except_clauses):
            except_clause = self.except_clauses[i] = except_clause.analyse_expressions(env)
6693 6694 6695 6696 6697
            if default_clause_seen:
                error(except_clause.pos, "default 'except:' must be last")
            if not except_clause.pattern:
                default_clause_seen = 1
        self.has_default_clause = default_clause_seen
William Stein's avatar
William Stein committed
6698
        if self.else_clause:
6699 6700
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6701

6702
    nogil_check = Node.gil_error
6703 6704
    gil_message = "Try-except statement"

William Stein's avatar
William Stein committed
6705
    def generate_execution_code(self, code):
6706
        old_return_label = code.return_label
6707
        old_break_label = code.break_label
6708
        old_continue_label = code.continue_label
William Stein's avatar
William Stein committed
6709 6710
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
6711 6712 6713
        except_end_label = code.new_label('exception_handled')
        except_error_label = code.new_label('except_error')
        except_return_label = code.new_label('except_return')
6714
        try_return_label = code.new_label('try_return')
6715 6716
        try_break_label = code.new_label('try_break') if old_break_label else None
        try_continue_label = code.new_label('try_continue') if old_continue_label else None
6717
        try_end_label = code.new_label('try_end')
6718

6719
        exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
6720
                         for _ in range(3)]
6721
        code.mark_pos(self.pos)
6722
        code.putln("{")
6723
        save_exc = code.insertion_point()
William Stein's avatar
William Stein committed
6724 6725
        code.putln(
            "/*try:*/ {")
6726
        code.return_label = try_return_label
6727
        code.break_label = try_break_label
6728
        code.continue_label = try_continue_label
William Stein's avatar
William Stein committed
6729
        self.body.generate_execution_code(code)
6730
        code.mark_pos(self.pos, trace=False)
William Stein's avatar
William Stein committed
6731 6732
        code.putln(
            "}")
6733
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
6734 6735 6736 6737
        can_raise = code.label_used(our_error_label)

        if can_raise:
            # inject code before the try block to save away the exception state
6738
            code.globalstate.use_utility_code(reset_exception_utility_code)
6739 6740 6741 6742
            save_exc.putln("__Pyx_PyThreadState_declare")
            save_exc.putln("__Pyx_PyThreadState_assign")
            save_exc.putln("__Pyx_ExceptionSave(%s);" % (
                ', '.join(['&%s' % var for var in exc_save_vars])))
6743 6744 6745 6746 6747 6748 6749 6750 6751 6752 6753
            for var in exc_save_vars:
                save_exc.put_xgotref(var)

            def restore_saved_exception():
                for name in exc_save_vars:
                    code.put_xgiveref(name)
                code.putln("__Pyx_ExceptionReset(%s);" %
                           ', '.join(exc_save_vars))
        else:
            # try block cannot raise exceptions, but we had to allocate the temps above,
            # so just keep the C compiler from complaining about them being unused
6754
            save_exc.putln("if (%s); else {/*mark used*/}" % '||'.join(exc_save_vars))
6755 6756 6757 6758

            def restore_saved_exception():
                pass

6759 6760
        code.error_label = except_error_label
        code.return_label = except_return_label
6761
        normal_case_terminates = self.body.is_terminator
William Stein's avatar
William Stein committed
6762
        if self.else_clause:
6763
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
6764 6765 6766 6767 6768
            code.putln(
                "/*else:*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln(
                "}")
6769 6770
            if not normal_case_terminates:
                normal_case_terminates = self.else_clause.is_terminator
6771

6772
        if can_raise:
6773 6774 6775 6776
            if not normal_case_terminates:
                for var in exc_save_vars:
                    code.put_xdecref_clear(var, py_object_type)
                code.put_goto(try_end_label)
6777
            code.put_label(our_error_label)
6778
            code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
6779 6780 6781 6782 6783 6784 6785 6786 6787 6788 6789 6790
            for temp_name, temp_type in temps_to_clean_up:
                code.put_xdecref_clear(temp_name, temp_type)
            for except_clause in self.except_clauses:
                except_clause.generate_handling_code(code, except_end_label)
            if not self.has_default_clause:
                code.put_goto(except_error_label)

        for exit_label, old_label in [(except_error_label, old_error_label),
                                      (try_break_label, old_break_label),
                                      (try_continue_label, old_continue_label),
                                      (try_return_label, old_return_label),
                                      (except_return_label, old_return_label)]:
6791
            if code.label_used(exit_label):
6792
                if not normal_case_terminates and not code.label_used(try_end_label):
6793
                    code.put_goto(try_end_label)
6794
                code.put_label(exit_label)
6795
                code.mark_pos(self.pos, trace=False)
6796 6797 6798
                if can_raise:
                    code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
                    restore_saved_exception()
6799
                code.put_goto(old_label)
6800 6801

        if code.label_used(except_end_label):
6802
            if not normal_case_terminates and not code.label_used(try_end_label):
6803
                code.put_goto(try_end_label)
6804
            code.put_label(except_end_label)
6805 6806 6807
            if can_raise:
                code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
                restore_saved_exception()
6808 6809
        if code.label_used(try_end_label):
            code.put_label(try_end_label)
6810 6811
        code.putln("}")

6812 6813 6814
        for cname in exc_save_vars:
            code.funcstate.release_temp(cname)

6815
        code.return_label = old_return_label
6816
        code.break_label = old_break_label
6817
        code.continue_label = old_continue_label
6818
        code.error_label = old_error_label
William Stein's avatar
William Stein committed
6819

6820 6821 6822 6823 6824 6825 6826
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        for except_clause in self.except_clauses:
            except_clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6827 6828 6829 6830 6831 6832 6833
    def annotate(self, code):
        self.body.annotate(code)
        for except_node in self.except_clauses:
            except_node.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
6834 6835 6836 6837

class ExceptClauseNode(Node):
    #  Part of try ... except statement.
    #
6838
    #  pattern        [ExprNode]
William Stein's avatar
William Stein committed
6839 6840
    #  target         ExprNode or None
    #  body           StatNode
6841
    #  excinfo_target TupleNode(3*ResultRefNode) or None   optional target for exception info (not owned here!)
William Stein's avatar
William Stein committed
6842 6843 6844
    #  match_flag     string             result of exception match
    #  exc_value      ExcValueNode       used internally
    #  function_name  string             qualified name of enclosing function
6845
    #  exc_vars       (string * 3)       local exception variables
6846
    #  is_except_as   bool               Py3-style "except ... as xyz"
6847 6848 6849 6850

    # excinfo_target is never set by the parser, but can be set by a transform
    # in order to extract more extensive information about the exception as a
    # sys.exc_info()-style tuple into a target variable
6851

6852
    child_attrs = ["pattern", "target", "body", "exc_value"]
6853

6854
    exc_value = None
6855
    excinfo_target = None
6856
    is_except_as = False
6857

William Stein's avatar
William Stein committed
6858 6859 6860 6861
    def analyse_declarations(self, env):
        if self.target:
            self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
6862

William Stein's avatar
William Stein committed
6863 6864 6865
    def analyse_expressions(self, env):
        self.function_name = env.qualified_name
        if self.pattern:
6866 6867
            # normalise/unpack self.pattern into a list
            for i, pattern in enumerate(self.pattern):
6868
                pattern = pattern.analyse_expressions(env)
6869
                self.pattern[i] = pattern.coerce_to_pyobject(env)
6870

William Stein's avatar
William Stein committed
6871
        if self.target:
6872
            from . import ExprNodes
6873
            self.exc_value = ExprNodes.ExcValueNode(self.pos)
6874
            self.target = self.target.analyse_target_expression(env, self.exc_value)
6875

6876 6877
        self.body = self.body.analyse_expressions(env)
        return self
6878

William Stein's avatar
William Stein committed
6879 6880 6881
    def generate_handling_code(self, code, end_label):
        code.mark_pos(self.pos)
        if self.pattern:
6882
            code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c"))
6883 6884 6885
            exc_tests = []
            for pattern in self.pattern:
                pattern.generate_evaluation_code(code)
6886
                exc_tests.append("__Pyx_PyErr_ExceptionMatches(%s)" % pattern.py_result())
6887

6888
            match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
William Stein's avatar
William Stein committed
6889
            code.putln(
6890 6891 6892 6893
                "%s = %s;" % (match_flag, ' || '.join(exc_tests)))
            for pattern in self.pattern:
                pattern.generate_disposal_code(code)
                pattern.free_temps(code)
William Stein's avatar
William Stein committed
6894 6895
            code.putln(
                "if (%s) {" %
6896 6897
                    match_flag)
            code.funcstate.release_temp(match_flag)
William Stein's avatar
William Stein committed
6898
        else:
6899
            code.putln("/*except:*/ {")
6900

6901 6902
        if (not getattr(self.body, 'stats', True)
                and self.excinfo_target is None
6903
                and self.target is None):
Stefan Behnel's avatar
Stefan Behnel committed
6904 6905
            # most simple case: no exception variable, empty body (pass)
            # => reset the exception state, done
6906 6907
            code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
            code.putln("__Pyx_ErrRestore(0,0,0);")
6908 6909 6910
            code.put_goto(end_label)
            code.putln("}")
            return
6911

6912 6913
        exc_vars = [code.funcstate.allocate_temp(py_object_type,
                                                 manage_ref=True)
6914
                    for _ in range(3)]
6915
        code.put_add_traceback(self.function_name)
William Stein's avatar
William Stein committed
6916
        # We always have to fetch the exception value even if
6917
        # there is no target, because this also normalises the
William Stein's avatar
William Stein committed
6918
        # exception and stores it in the thread state.
6919 6920
        code.globalstate.use_utility_code(get_exception_utility_code)
        exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
6921 6922
        code.putln("if (__Pyx_GetException(%s) < 0) %s" % (
            exc_args, code.error_goto(self.pos)))
6923
        for x in exc_vars:
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
6924
            code.put_gotref(x)
William Stein's avatar
William Stein committed
6925
        if self.target:
6926
            self.exc_value.set_var(exc_vars[1])
6927
            self.exc_value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
6928
            self.target.generate_assignment_code(self.exc_value, code)
6929
        if self.excinfo_target is not None:
6930
            for tempvar, node in zip(exc_vars, self.excinfo_target.args):
6931
                node.set_var(tempvar)
6932

6933 6934 6935 6936
        old_break_label, old_continue_label = code.break_label, code.continue_label
        code.break_label = code.new_label('except_break')
        code.continue_label = code.new_label('except_continue')

6937
        old_exc_vars = code.funcstate.exc_vars
6938
        code.funcstate.exc_vars = exc_vars
William Stein's avatar
William Stein committed
6939
        self.body.generate_execution_code(code)
6940
        code.funcstate.exc_vars = old_exc_vars
6941 6942 6943 6944
        if not self.body.is_terminator:
            for var in exc_vars:
                code.put_decref_clear(var, py_object_type)
            code.put_goto(end_label)
6945

Stefan Behnel's avatar
Stefan Behnel committed
6946 6947 6948 6949 6950 6951 6952
        for new_label, old_label in [(code.break_label, old_break_label),
                                     (code.continue_label, old_continue_label)]:
            if code.label_used(new_label):
                code.put_label(new_label)
                for var in exc_vars:
                    code.put_decref_clear(var, py_object_type)
                code.put_goto(old_label)
Robert Bradshaw's avatar
Robert Bradshaw committed
6953
        code.break_label = old_break_label
6954
        code.continue_label = old_continue_label
6955

6956 6957
        for temp in exc_vars:
            code.funcstate.release_temp(temp)
6958

William Stein's avatar
William Stein committed
6959 6960 6961
        code.putln(
            "}")

6962 6963 6964 6965
    def generate_function_definitions(self, env, code):
        if self.target is not None:
            self.target.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
6966

6967
    def annotate(self, code):
6968
        if self.pattern:
6969 6970
            for pattern in self.pattern:
                pattern.annotate(code)
6971 6972 6973 6974
        if self.target:
            self.target.annotate(code)
        self.body.annotate(code)

William Stein's avatar
William Stein committed
6975 6976 6977 6978 6979 6980

class TryFinallyStatNode(StatNode):
    #  try ... finally statement
    #
    #  body             StatNode
    #  finally_clause   StatNode
6981
    #  finally_except_clause  deep-copy of finally_clause for exception case
6982
    #
Stefan Behnel's avatar
Stefan Behnel committed
6983 6984
    #  Each of the continue, break, return and error gotos runs
    #  into its own deep-copy of the finally block code.
William Stein's avatar
William Stein committed
6985 6986 6987
    #  In addition, if we're doing an error, we save the
    #  exception on entry to the finally block and restore
    #  it on exit.
6988

6989
    child_attrs = ["body", "finally_clause", "finally_except_clause"]
6990

6991
    preserve_exception = 1
6992

6993 6994
    # handle exception case, in addition to return/break/continue
    handle_error_case = True
6995
    func_return_type = None
6996
    finally_except_clause = None
6997

6998 6999
    is_try_finally_in_nogil = False

7000
    @staticmethod
7001 7002 7003
    def create_analysed(pos, env, body, finally_clause):
        node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
        return node
7004

William Stein's avatar
William Stein committed
7005 7006
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
7007 7008
        self.finally_except_clause = copy.deepcopy(self.finally_clause)
        self.finally_except_clause.analyse_declarations(env)
William Stein's avatar
William Stein committed
7009
        self.finally_clause.analyse_declarations(env)
7010

William Stein's avatar
William Stein committed
7011
    def analyse_expressions(self, env):
7012 7013
        self.body = self.body.analyse_expressions(env)
        self.finally_clause = self.finally_clause.analyse_expressions(env)
7014
        self.finally_except_clause = self.finally_except_clause.analyse_expressions(env)
Stefan Behnel's avatar
Stefan Behnel committed
7015 7016
        if env.return_type and not env.return_type.is_void:
            self.func_return_type = env.return_type
7017
        return self
7018

7019
    nogil_check = Node.gil_error
7020 7021
    gil_message = "Try-finally statement"

William Stein's avatar
William Stein committed
7022
    def generate_execution_code(self, code):
7023
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
7024 7025 7026 7027
        old_error_label = code.error_label
        old_labels = code.all_new_labels()
        new_labels = code.get_all_labels()
        new_error_label = code.error_label
7028 7029
        if not self.handle_error_case:
            code.error_label = old_error_label
William Stein's avatar
William Stein committed
7030
        catch_label = code.new_label()
7031 7032

        code.putln("/*try:*/ {")
Stefan Behnel's avatar
Stefan Behnel committed
7033 7034
        was_in_try_finally = code.funcstate.in_try_finally
        code.funcstate.in_try_finally = 1
7035

William Stein's avatar
William Stein committed
7036
        self.body.generate_execution_code(code)
7037

Stefan Behnel's avatar
Stefan Behnel committed
7038
        code.funcstate.in_try_finally = was_in_try_finally
7039
        code.putln("}")
7040
        code.set_all_labels(old_labels)
7041

7042
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
7043
        code.mark_pos(self.finally_clause.pos)
7044 7045
        code.putln("/*finally:*/ {")

7046 7047 7048 7049 7050 7051 7052 7053 7054
        def fresh_finally_clause(_next=[self.finally_clause]):
            # generate the original subtree once and always keep a fresh copy
            node = _next[0]
            node_copy = copy.deepcopy(node)
            if node is self.finally_clause:
                _next[0] = node_copy
            else:
                node = node_copy
            return node
7055 7056

        preserve_error = self.preserve_exception and code.label_used(new_error_label)
7057
        needs_success_cleanup = not self.finally_clause.is_terminator
7058 7059 7060

        if not self.body.is_terminator:
            code.putln('/*normal exit:*/{')
7061 7062
            fresh_finally_clause().generate_execution_code(code)
            if not self.finally_clause.is_terminator:
7063
                code.put_goto(catch_label)
7064
            code.putln('}')
7065

7066 7067
        if preserve_error:
            code.putln('/*exception exit:*/{')
7068
            code.putln("__Pyx_PyThreadState_declare")
7069 7070
            if self.is_try_finally_in_nogil:
                code.declare_gilstate()
7071 7072 7073 7074 7075 7076 7077 7078 7079 7080 7081
            if needs_success_cleanup:
                exc_lineno_cnames = tuple([
                    code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
                    for _ in range(2)])
                exc_filename_cname = code.funcstate.allocate_temp(
                    PyrexTypes.CPtrType(PyrexTypes.c_const_type(PyrexTypes.c_char_type)),
                    manage_ref=False)
            else:
                exc_lineno_cnames = exc_filename_cname = None
            exc_vars = tuple([
                code.funcstate.allocate_temp(py_object_type, manage_ref=False)
7082
                for _ in range(6)])
7083
            code.put_label(new_error_label)
7084
            self.put_error_catcher(
7085
                code, temps_to_clean_up, exc_vars, exc_lineno_cnames, exc_filename_cname)
7086
            finally_old_labels = code.all_new_labels()
7087

7088
            code.putln('{')
7089 7090
            old_exc_vars = code.funcstate.exc_vars
            code.funcstate.exc_vars = exc_vars[:3]
7091
            self.finally_except_clause.generate_execution_code(code)
7092
            code.funcstate.exc_vars = old_exc_vars
7093 7094
            code.putln('}')

7095
            if needs_success_cleanup:
7096 7097 7098 7099 7100 7101
                self.put_error_uncatcher(code, exc_vars, exc_lineno_cnames, exc_filename_cname)
                if exc_lineno_cnames:
                    for cname in exc_lineno_cnames:
                        code.funcstate.release_temp(cname)
                if exc_filename_cname:
                    code.funcstate.release_temp(exc_filename_cname)
7102
                code.put_goto(old_error_label)
7103

7104 7105 7106 7107
            for new_label, old_label in zip(code.get_all_labels(), finally_old_labels):
                if not code.label_used(new_label):
                    continue
                code.put_label(new_label)
7108
                self.put_error_cleaner(code, exc_vars)
7109
                code.put_goto(old_label)
7110 7111 7112

            for cname in exc_vars:
                code.funcstate.release_temp(cname)
7113
            code.putln('}')
7114

7115
        code.set_all_labels(old_labels)
7116
        return_label = code.return_label
7117 7118 7119 7120 7121
        for i, (new_label, old_label) in enumerate(zip(new_labels, old_labels)):
            if not code.label_used(new_label):
                continue
            if new_label == new_error_label and preserve_error:
                continue  # handled above
7122

7123 7124
            code.put('%s: ' % new_label)
            code.putln('{')
7125 7126 7127
            ret_temp = None
            if old_label == return_label and not self.finally_clause.is_terminator:
                # store away return value for later reuse
7128 7129 7130
                if (self.func_return_type and
                        not self.is_try_finally_in_nogil and
                        not isinstance(self.finally_clause, GILExitNode)):
7131 7132 7133 7134 7135
                    ret_temp = code.funcstate.allocate_temp(
                        self.func_return_type, manage_ref=False)
                    code.putln("%s = %s;" % (ret_temp, Naming.retval_cname))
                    if self.func_return_type.is_pyobject:
                        code.putln("%s = 0;" % Naming.retval_cname)
7136
            fresh_finally_clause().generate_execution_code(code)
7137 7138 7139 7140 7141 7142
            if ret_temp:
                code.putln("%s = %s;" % (Naming.retval_cname, ret_temp))
                if self.func_return_type.is_pyobject:
                    code.putln("%s = 0;" % ret_temp)
                code.funcstate.release_temp(ret_temp)
                ret_temp = None
7143
            if not self.finally_clause.is_terminator:
7144 7145
                code.put_goto(old_label)
            code.putln('}')
7146 7147

        # End finally
7148
        code.put_label(catch_label)
William Stein's avatar
William Stein committed
7149 7150 7151
        code.putln(
            "}")

7152 7153 7154 7155
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        self.finally_clause.generate_function_definitions(env, code)

7156 7157
    def put_error_catcher(self, code, temps_to_clean_up, exc_vars,
                          exc_lineno_cnames, exc_filename_cname):
7158
        code.globalstate.use_utility_code(restore_exception_utility_code)
7159 7160
        code.globalstate.use_utility_code(get_exception_utility_code)
        code.globalstate.use_utility_code(swap_exception_utility_code)
7161

7162
        code.putln(' '.join(["%s = 0;"]*len(exc_vars)) % exc_vars)
7163 7164
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7165
        code.putln("__Pyx_PyThreadState_assign")
7166

7167 7168
        for temp_name, type in temps_to_clean_up:
            code.put_xdecref_clear(temp_name, type)
7169

7170 7171 7172 7173 7174 7175 7176 7177 7178
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3)"
                   " __Pyx_ExceptionSwap(&%s, &%s, &%s);" % exc_vars[3:])
        code.putln("if ((PY_MAJOR_VERSION < 3) ||"
                   # if __Pyx_GetException() fails in Py3,
                   # store the newly raised exception instead
                   " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) "
                   "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2))
7179 7180 7181 7182 7183 7184 7185
        for var in exc_vars:
            code.put_xgotref(var)
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                exc_lineno_cnames[0], Naming.lineno_cname,
                exc_lineno_cnames[1], Naming.clineno_cname,
                exc_filename_cname, Naming.filename_cname))
7186 7187 7188 7189

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

7190
    def put_error_uncatcher(self, code, exc_vars, exc_lineno_cnames, exc_filename_cname):
7191
        code.globalstate.use_utility_code(restore_exception_utility_code)
7192
        code.globalstate.use_utility_code(reset_exception_utility_code)
7193 7194 7195

        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7196
        code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
7197

7198 7199 7200
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
7201 7202 7203
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7204
        code.putln("}")
7205
        for var in exc_vars[:3]:
7206
            code.put_xgiveref(var)
7207
        code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3])
7208 7209 7210 7211

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

7212
        code.putln(' '.join(["%s = 0;"]*len(exc_vars)) % exc_vars)
7213 7214 7215 7216 7217
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                Naming.lineno_cname, exc_lineno_cnames[0],
                Naming.clineno_cname, exc_lineno_cnames[1],
                Naming.filename_cname, exc_filename_cname))
7218

7219
    def put_error_cleaner(self, code, exc_vars):
7220
        code.globalstate.use_utility_code(reset_exception_utility_code)
7221 7222
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7223 7224
        code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded

7225 7226 7227
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
7228 7229 7230
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7231
        code.putln("}")
7232
        for var in exc_vars[:3]:
7233
            code.put_xdecref_clear(var, py_object_type)
7234 7235
        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()
7236
        code.putln(' '.join(["%s = 0;"]*3) % exc_vars[3:])
William Stein's avatar
William Stein committed
7237

7238 7239 7240 7241
    def annotate(self, code):
        self.body.annotate(code)
        self.finally_clause.annotate(code)

William Stein's avatar
William Stein committed
7242

7243 7244 7245 7246 7247 7248 7249 7250 7251 7252
class NogilTryFinallyStatNode(TryFinallyStatNode):
    """
    A try/finally statement that may be used in nogil code sections.
    """

    preserve_exception = False
    nogil_check = None


class GILStatNode(NogilTryFinallyStatNode):
7253 7254 7255
    #  'with gil' or 'with nogil' statement
    #
    #   state   string   'gil' or 'nogil'
7256

7257 7258
    state_temp = None

7259 7260
    def __init__(self, pos, state, body):
        self.state = state
7261
        self.create_state_temp_if_needed(pos, state, body)
7262 7263
        TryFinallyStatNode.__init__(
            self, pos,
7264 7265 7266 7267 7268
            body=body,
            finally_clause=GILExitNode(
                pos, state=state, state_temp=self.state_temp))

    def create_state_temp_if_needed(self, pos, state, body):
7269
        from .ParseTreeTransforms import YieldNodeCollector
7270 7271
        collector = YieldNodeCollector()
        collector.visitchildren(body)
7272
        if not collector.yields and not collector.awaits:
7273 7274
            return

7275 7276 7277 7278
        if state == 'gil':
            temp_type = PyrexTypes.c_gilstate_type
        else:
            temp_type = PyrexTypes.c_threadstate_ptr_type
7279
        from . import ExprNodes
7280
        self.state_temp = ExprNodes.TempNode(pos, temp_type)
7281

7282 7283 7284 7285
    def analyse_declarations(self, env):
        env._in_with_gil_block = (self.state == 'gil')
        if self.state == 'gil':
            env.has_with_gil_block = True
7286

7287 7288
        return super(GILStatNode, self).analyse_declarations(env)

7289
    def analyse_expressions(self, env):
7290 7291
        env.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
7292
        was_nogil = env.nogil
7293
        env.nogil = self.state == 'nogil'
7294
        node = TryFinallyStatNode.analyse_expressions(self, env)
7295
        env.nogil = was_nogil
7296
        return node
7297

7298
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
7299
        code.mark_pos(self.pos)
7300
        code.begin_block()
7301 7302 7303 7304 7305
        if self.state_temp:
            self.state_temp.allocate(code)
            variable = self.state_temp.result()
        else:
            variable = None
7306

7307
        old_gil_config = code.funcstate.gil_owned
7308
        if self.state == 'gil':
7309
            code.put_ensure_gil(variable=variable)
7310
            code.funcstate.gil_owned = True
7311
        else:
7312
            code.put_release_gil(variable=variable)
7313
            code.funcstate.gil_owned = False
7314

7315
        TryFinallyStatNode.generate_execution_code(self, code)
7316

7317 7318
        if self.state_temp:
            self.state_temp.release(code)
7319

7320
        code.funcstate.gil_owned = old_gil_config
7321
        code.end_block()
7322 7323 7324


class GILExitNode(StatNode):
7325 7326 7327 7328 7329
    """
    Used as the 'finally' block in a GILStatNode

    state   string   'gil' or 'nogil'
    """
7330

7331
    child_attrs = []
7332
    state_temp = None
7333

7334
    def analyse_expressions(self, env):
7335
        return self
7336 7337

    def generate_execution_code(self, code):
7338 7339 7340 7341 7342
        if self.state_temp:
            variable = self.state_temp.result()
        else:
            variable = None

7343
        if self.state == 'gil':
7344
            code.put_release_ensured_gil(variable)
7345
        else:
7346
            code.put_acquire_gil(variable)
7347 7348


7349 7350 7351 7352 7353 7354 7355
class EnsureGILNode(GILExitNode):
    """
    Ensure the GIL in nogil functions for cleanup before returning.
    """

    def generate_execution_code(self, code):
        code.put_ensure_gil(declare_gilstate=False)
7356

7357

7358 7359 7360 7361 7362
def cython_view_utility_code():
    from . import MemoryView
    return MemoryView.view_utility_code


7363 7364 7365
utility_code_for_cimports = {
    # utility code (or inlining c) in a pxd (or pyx) file.
    # TODO: Consider a generic user-level mechanism for importing
7366 7367 7368
    'cpython.array'         : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
    'cpython.array.array'   : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
    'cython.view'           : cython_view_utility_code,
7369
}
7370

7371 7372 7373
utility_code_for_imports = {
    # utility code used when special modules are imported.
    # TODO: Consider a generic user-level mechanism for importing
7374 7375
    'asyncio': ("__Pyx_patch_asyncio", "PatchAsyncIO", "Coroutine.c"),
    'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"),
7376 7377 7378
}


William Stein's avatar
William Stein committed
7379 7380 7381 7382 7383
class CImportStatNode(StatNode):
    #  cimport statement
    #
    #  module_name   string           Qualified name of module being imported
    #  as_name       string or None   Name specified in "as" clause, if any
7384
    #  is_absolute   bool             True for absolute imports, False otherwise
7385 7386

    child_attrs = []
7387
    is_absolute = False
7388

William Stein's avatar
William Stein committed
7389
    def analyse_declarations(self, env):
7390 7391 7392
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
7393 7394
        module_scope = env.find_module(
            self.module_name, self.pos, relative_level=0 if self.is_absolute else -1)
William Stein's avatar
William Stein committed
7395
        if "." in self.module_name:
7396
            names = [EncodedString(name) for name in self.module_name.split(".")]
William Stein's avatar
William Stein committed
7397 7398 7399 7400 7401 7402 7403 7404 7405 7406
            top_name = names[0]
            top_module_scope = env.context.find_submodule(top_name)
            module_scope = top_module_scope
            for name in names[1:]:
                submodule_scope = module_scope.find_submodule(name)
                module_scope.declare_module(name, submodule_scope, self.pos)
                module_scope = submodule_scope
            if self.as_name:
                env.declare_module(self.as_name, module_scope, self.pos)
            else:
7407
                env.add_imported_module(module_scope)
William Stein's avatar
William Stein committed
7408 7409 7410 7411
                env.declare_module(top_name, top_module_scope, self.pos)
        else:
            name = self.as_name or self.module_name
            env.declare_module(name, module_scope, self.pos)
7412
        if self.module_name in utility_code_for_cimports:
7413
            env.use_utility_code(utility_code_for_cimports[self.module_name]())
William Stein's avatar
William Stein committed
7414 7415

    def analyse_expressions(self, env):
7416
        return self
7417

William Stein's avatar
William Stein committed
7418 7419
    def generate_execution_code(self, code):
        pass
7420

William Stein's avatar
William Stein committed
7421 7422 7423 7424

class FromCImportStatNode(StatNode):
    #  from ... cimport statement
    #
7425
    #  module_name     string                        Qualified name of module
7426
    #  relative_level  int or None                   Relative import: number of dots before module_name
7427
    #  imported_names  [(pos, name, as_name, kind)]  Names to be imported
7428

7429
    child_attrs = []
7430 7431 7432
    module_name = None
    relative_level = None
    imported_names = None
7433

William Stein's avatar
William Stein committed
7434
    def analyse_declarations(self, env):
7435 7436 7437
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
7438 7439
        if self.relative_level and self.relative_level > env.qualified_name.count('.'):
            error(self.pos, "relative cimport beyond main package is not allowed")
7440
            return
7441 7442
        module_scope = env.find_module(self.module_name, self.pos, relative_level=self.relative_level)
        module_name = module_scope.qualified_name
William Stein's avatar
William Stein committed
7443
        env.add_imported_module(module_scope)
7444
        for pos, name, as_name, kind in self.imported_names:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7445
            if name == "*":
7446
                for local_name, entry in list(module_scope.entries.items()):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7447 7448
                    env.add_imported_entry(local_name, entry, pos)
            else:
7449 7450 7451 7452
                entry = module_scope.lookup(name)
                if entry:
                    if kind and not self.declaration_matches(entry, kind):
                        entry.redeclared(pos)
7453
                    entry.used = 1
7454 7455
                else:
                    if kind == 'struct' or kind == 'union':
7456 7457
                        entry = module_scope.declare_struct_or_union(
                            name, kind=kind, scope=None, typedef_flag=0, pos=pos)
7458
                    elif kind == 'class':
7459
                        entry = module_scope.declare_c_class(name, pos=pos, module_name=module_name)
7460
                    else:
7461 7462
                        submodule_scope = env.context.find_module(
                            name, relative_to=module_scope, pos=self.pos, absolute_fallback=False)
7463 7464 7465
                        if submodule_scope.parent_module is module_scope:
                            env.declare_module(as_name or name, submodule_scope, self.pos)
                        else:
7466
                            error(pos, "Name '%s' not declared in module '%s'" % (name, module_name))
7467

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7468 7469 7470
                if entry:
                    local_name = as_name or name
                    env.add_imported_entry(local_name, entry, pos)
7471

7472
        if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now
7473
            if module_name in utility_code_for_cimports:
7474
                env.use_utility_code(utility_code_for_cimports[module_name]())
7475
            for _, name, _, _ in self.imported_names:
7476
                fqname = '%s.%s' % (module_name, name)
7477
                if fqname in utility_code_for_cimports:
7478
                    env.use_utility_code(utility_code_for_cimports[fqname]())
7479

7480
    def declaration_matches(self, entry, kind):
7481 7482 7483 7484 7485 7486 7487 7488 7489 7490 7491 7492
        if not entry.is_type:
            return 0
        type = entry.type
        if kind == 'class':
            if not type.is_extension_type:
                return 0
        else:
            if not type.is_struct_or_union:
                return 0
            if kind != type.kind:
                return 0
        return 1
William Stein's avatar
William Stein committed
7493 7494

    def analyse_expressions(self, env):
7495
        return self
7496

William Stein's avatar
William Stein committed
7497 7498 7499 7500 7501 7502 7503 7504 7505
    def generate_execution_code(self, code):
        pass


class FromImportStatNode(StatNode):
    #  from ... import statement
    #
    #  module           ImportNode
    #  items            [(string, NameNode)]
7506
    #  interned_items   [(string, NameNode, ExprNode)]
William Stein's avatar
William Stein committed
7507
    #  item             PyTempNode            used internally
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7508
    #  import_star      boolean               used internally
7509 7510

    child_attrs = ["module"]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7511
    import_star = 0
7512

William Stein's avatar
William Stein committed
7513
    def analyse_declarations(self, env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7514 7515 7516 7517 7518 7519 7520 7521 7522
        for name, target in self.items:
            if name == "*":
                if not env.is_module_scope:
                    error(self.pos, "import * only allowed at module level")
                    return
                env.has_import_star = 1
                self.import_star = 1
            else:
                target.analyse_target_declaration(env)
7523

William Stein's avatar
William Stein committed
7524
    def analyse_expressions(self, env):
7525
        from . import ExprNodes
7526
        self.module = self.module.analyse_expressions(env)
7527
        self.item = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
William Stein's avatar
William Stein committed
7528 7529
        self.interned_items = []
        for name, target in self.items:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7530 7531 7532
            if name == '*':
                for _, entry in env.entries.items():
                    if not entry.is_type and entry.type.is_extension_type:
7533
                        env.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7534 7535
                        break
            else:
7536
                entry = env.lookup(target.name)
7537 7538
                # check whether or not entry is already cimported
                if (entry.is_type and entry.type.name == name
Stefan Behnel's avatar
Stefan Behnel committed
7539
                        and hasattr(entry.type, 'module_name')):
7540 7541 7542 7543 7544
                    if entry.type.module_name == self.module.module_name.value:
                        # cimported with absolute name
                        continue
                    try:
                        # cimported with relative name
7545 7546
                        module = env.find_module(self.module.module_name.value, pos=self.pos,
                                                 relative_level=self.module.level)
7547 7548 7549
                        if entry.type.module_name == module.qualified_name:
                            continue
                    except AttributeError:
7550
                        pass
7551
                target = target.analyse_target_expression(env, None)  # FIXME?
7552 7553 7554 7555
                if target.type is py_object_type:
                    coerced_item = None
                else:
                    coerced_item = self.item.coerce_to(target.type, env)
7556
                self.interned_items.append((name, target, coerced_item))
7557
        return self
7558

William Stein's avatar
William Stein committed
7559
    def generate_execution_code(self, code):
7560
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
7561
        self.module.generate_evaluation_code(code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7562 7563 7564 7565 7566 7567
        if self.import_star:
            code.putln(
                'if (%s(%s) < 0) %s;' % (
                    Naming.import_star,
                    self.module.py_result(),
                    code.error_goto(self.pos)))
7568 7569
        item_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.item.set_cname(item_temp)
7570 7571 7572
        if self.interned_items:
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ImportFrom", "ImportExport.c"))
7573
        for name, target, coerced_item in self.interned_items:
7574
            code.putln(
7575
                '%s = __Pyx_ImportFrom(%s, %s); %s' % (
7576
                    item_temp,
7577
                    self.module.py_result(),
7578 7579
                    code.intern_identifier(name),
                    code.error_goto_if_null(item_temp, self.pos)))
7580
            code.put_gotref(item_temp)
7581 7582 7583 7584 7585 7586
            if coerced_item is None:
                target.generate_assignment_code(self.item, code)
            else:
                coerced_item.allocate_temp_result(code)
                coerced_item.generate_result_code(code)
                target.generate_assignment_code(coerced_item, code)
7587 7588
            code.put_decref_clear(item_temp, py_object_type)
        code.funcstate.release_temp(item_temp)
William Stein's avatar
William Stein committed
7589
        self.module.generate_disposal_code(code)
7590
        self.module.free_temps(code)
William Stein's avatar
William Stein committed
7591

7592

Mark Florisson's avatar
Mark Florisson committed
7593 7594 7595 7596 7597 7598 7599 7600 7601 7602
class ParallelNode(Node):
    """
    Base class for cython.parallel constructs.
    """

    nogil_check = None


class ParallelStatNode(StatNode, ParallelNode):
    """
7603
    Base class for 'with cython.parallel.parallel():' and 'for i in prange():'.
Mark Florisson's avatar
Mark Florisson committed
7604 7605 7606 7607 7608

    assignments     { Entry(var) : (var.pos, inplace_operator_or_None) }
                    assignments to variables in this parallel section

    parent          parent ParallelStatNode or None
7609 7610 7611
    is_parallel     indicates whether this node is OpenMP parallel
                    (true for #pragma omp parallel for and
                              #pragma omp parallel)
Mark Florisson's avatar
Mark Florisson committed
7612 7613 7614 7615 7616 7617 7618 7619 7620 7621 7622

    is_parallel is true for:

        #pragma omp parallel
        #pragma omp parallel for

    sections, but NOT for

        #pragma omp for

    We need this to determine the sharing attributes.
7623 7624 7625

    privatization_insertion_point   a code insertion point used to make temps
                                    private (esp. the "nsteps" temp)
7626 7627 7628 7629

    args         tuple          the arguments passed to the parallel construct
    kwargs       DictNode       the keyword arguments passed to the parallel
                                construct (replaced by its compile time value)
Mark Florisson's avatar
Mark Florisson committed
7630 7631
    """

7632
    child_attrs = ['body', 'num_threads']
Mark Florisson's avatar
Mark Florisson committed
7633 7634 7635 7636

    body = None

    is_prange = False
7637
    is_nested_prange = False
Mark Florisson's avatar
Mark Florisson committed
7638

7639
    error_label_used = False
7640

7641
    num_threads = None
7642
    chunksize = None
7643

7644 7645 7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660
    parallel_exc = (
        Naming.parallel_exc_type,
        Naming.parallel_exc_value,
        Naming.parallel_exc_tb,
    )

    parallel_pos_info = (
        Naming.parallel_filename,
        Naming.parallel_lineno,
        Naming.parallel_clineno,
    )

    pos_info = (
        Naming.filename_cname,
        Naming.lineno_cname,
        Naming.clineno_cname,
    )
7661

7662 7663
    critical_section_counter = 0

Mark Florisson's avatar
Mark Florisson committed
7664 7665
    def __init__(self, pos, **kwargs):
        super(ParallelStatNode, self).__init__(pos, **kwargs)
7666 7667

        # All assignments in this scope
Mark Florisson's avatar
Mark Florisson committed
7668 7669
        self.assignments = kwargs.get('assignments') or {}

7670 7671 7672 7673
        # All seen closure cnames and their temporary cnames
        self.seen_closure_vars = set()

        # Dict of variables that should be declared (first|last|)private or
7674 7675
        # reduction { Entry: (op, lastprivate) }.
        # If op is not None, it's a reduction.
7676
        self.privates = {}
Mark Florisson's avatar
Mark Florisson committed
7677

Mark Florisson's avatar
Mark Florisson committed
7678 7679 7680
        # [NameNode]
        self.assigned_nodes = []

Mark Florisson's avatar
Mark Florisson committed
7681 7682 7683
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)

7684 7685
        self.num_threads = None

7686
        if self.kwargs:
7687 7688 7689
            # Try to find num_threads and chunksize keyword arguments
            pairs = []
            for dictitem in self.kwargs.key_value_pairs:
7690 7691
                if dictitem.key.value == 'num_threads':
                    self.num_threads = dictitem.value
7692 7693 7694 7695 7696 7697
                elif self.is_prange and dictitem.key.value == 'chunksize':
                    self.chunksize = dictitem.value
                else:
                    pairs.append(dictitem)

            self.kwargs.key_value_pairs = pairs
7698

7699 7700
            try:
                self.kwargs = self.kwargs.compile_time_value(env)
7701
            except Exception as e:
7702 7703
                error(self.kwargs.pos, "Only compile-time values may be "
                                       "supplied as keyword arguments")
7704 7705 7706
        else:
            self.kwargs = {}

7707
        for kw, val in self.kwargs.items():
7708 7709 7710 7711 7712
            if kw not in self.valid_keyword_arguments:
                error(self.pos, "Invalid keyword argument: %s" % kw)
            else:
                setattr(self, kw, val)

7713
    def analyse_expressions(self, env):
7714
        if self.num_threads:
7715
            self.num_threads = self.num_threads.analyse_expressions(env)
7716 7717

        if self.chunksize:
7718
            self.chunksize = self.chunksize.analyse_expressions(env)
7719

7720
        self.body = self.body.analyse_expressions(env)
7721
        self.analyse_sharing_attributes(env)
7722

7723
        if self.num_threads is not None:
7724 7725
            if self.parent and self.parent.num_threads is not None and not self.parent.is_prange:
                error(self.pos, "num_threads already declared in outer section")
7726
            elif self.parent and not self.parent.is_prange:
7727
                error(self.pos, "num_threads must be declared in the parent parallel section")
7728
            elif (self.num_threads.type.is_int and
7729 7730 7731
                    self.num_threads.is_literal and
                    self.num_threads.compile_time_value(env) <= 0):
                error(self.pos, "argument to num_threads must be greater than 0")
7732

Mark Florisson's avatar
Mark Florisson committed
7733 7734
            if not self.num_threads.is_simple():
                self.num_threads = self.num_threads.coerce_to(
7735 7736
                    PyrexTypes.c_int_type, env).coerce_to_temp(env)
        return self
7737 7738

    def analyse_sharing_attributes(self, env):
Mark Florisson's avatar
Mark Florisson committed
7739
        """
7740 7741 7742
        Analyse the privates for this block and set them in self.privates.
        This should be called in a post-order fashion during the
        analyse_expressions phase
Mark Florisson's avatar
Mark Florisson committed
7743
        """
7744
        for entry, (pos, op) in self.assignments.items():
Mark Florisson's avatar
Mark Florisson committed
7745

7746 7747 7748 7749 7750
            if self.is_prange and not self.is_parallel:
                # closely nested prange in a with parallel block, disallow
                # assigning to privates in the with parallel block (we
                # consider it too implicit and magicky for users)
                if entry in self.parent.assignments:
7751
                    error(pos, "Cannot assign to private of outer parallel block")
7752 7753 7754 7755 7756 7757 7758
                    continue

            if not self.is_prange and op:
                # Again possible, but considered to magicky
                error(pos, "Reductions not allowed for parallel blocks")
                continue

Mark Florisson's avatar
Mark Florisson committed
7759 7760 7761
            # By default all variables should have the same values as if
            # executed sequentially
            lastprivate = True
7762
            self.propagate_var_privatization(entry, pos, op, lastprivate)
7763

7764
    def propagate_var_privatization(self, entry, pos, op, lastprivate):
7765 7766 7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792
        """
        Propagate the sharing attributes of a variable. If the privatization is
        determined by a parent scope, done propagate further.

        If we are a prange, we propagate our sharing attributes outwards to
        other pranges. If we are a prange in parallel block and the parallel
        block does not determine the variable private, we propagate to the
        parent of the parent. Recursion stops at parallel blocks, as they have
        no concept of lastprivate or reduction.

        So the following cases propagate:

            sum is a reduction for all loops:

                for i in prange(n):
                    for j in prange(n):
                        for k in prange(n):
                            sum += i * j * k

            sum is a reduction for both loops, local_var is private to the
            parallel with block:

                for i in prange(n):
                    with parallel:
                        local_var = ... # private to the parallel
                        for j in prange(n):
                            sum += i * j

Mark Florisson's avatar
Mark Florisson committed
7793 7794
        Nested with parallel blocks are disallowed, because they wouldn't
        allow you to propagate lastprivates or reductions:
7795 7796 7797 7798

            #pragma omp parallel for lastprivate(i)
            for i in prange(n):

Mark Florisson's avatar
Mark Florisson committed
7799 7800
                sum = 0

7801 7802 7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816
                #pragma omp parallel private(j, sum)
                with parallel:

                    #pragma omp parallel
                    with parallel:

                        #pragma omp for lastprivate(j) reduction(+:sum)
                        for j in prange(n):
                            sum += i

                    # sum and j are well-defined here

                # sum and j are undefined here

            # sum and j are undefined here
        """
7817
        self.privates[entry] = (op, lastprivate)
7818 7819 7820 7821 7822

        if entry.type.is_memoryviewslice:
            error(pos, "Memoryview slices can only be shared in parallel sections")
            return

7823 7824 7825 7826 7827 7828
        if self.is_prange:
            if not self.is_parallel and entry not in self.parent.assignments:
                # Parent is a parallel with block
                parent = self.parent.parent
            else:
                parent = self.parent
Mark Florisson's avatar
Mark Florisson committed
7829

7830 7831 7832
            # We don't need to propagate privates, only reductions and
            # lastprivates
            if parent and (op or lastprivate):
7833
                parent.propagate_var_privatization(entry, pos, op, lastprivate)
Mark Florisson's avatar
Mark Florisson committed
7834 7835 7836 7837 7838 7839 7840 7841 7842

    def _allocate_closure_temp(self, code, entry):
        """
        Helper function that allocate a temporary for a closure variable that
        is assigned to.
        """
        if self.parent:
            return self.parent._allocate_closure_temp(code, entry)

7843 7844 7845
        if entry.cname in self.seen_closure_vars:
            return entry.cname

7846
        cname = code.funcstate.allocate_temp(entry.type, True)
7847 7848 7849 7850 7851 7852

        # Add both the actual cname and the temp cname, as the actual cname
        # will be replaced with the temp cname on the entry
        self.seen_closure_vars.add(entry.cname)
        self.seen_closure_vars.add(cname)

Mark Florisson's avatar
Mark Florisson committed
7853 7854 7855 7856
        self.modified_entries.append((entry, entry.cname))
        code.putln("%s = %s;" % (cname, entry.cname))
        entry.cname = cname

7857
    def initialize_privates_to_nan(self, code, exclude=None):
7858
        first = True
7859

7860
        for entry, (op, lastprivate) in sorted(self.privates.items()):
7861
            if not op and (not exclude or entry != exclude):
7862
                invalid_value = entry.type.invalid_value()
7863

7864
                if invalid_value:
7865 7866 7867 7868
                    if first:
                        code.putln("/* Initialize private variables to "
                                   "invalid values */")
                        first = False
7869
                    code.putln("%s = %s;" % (entry.cname,
7870
                                             entry.type.cast_code(invalid_value)))
7871

7872
    def evaluate_before_block(self, code, expr):
Mark Florisson's avatar
Mark Florisson committed
7873
        c = self.begin_of_parallel_control_block_point_after_decls
7874 7875 7876 7877 7878 7879 7880 7881 7882 7883
        # we need to set the owner to ourselves temporarily, as
        # allocate_temp may generate a comment in the middle of our pragma
        # otherwise when DebugFlags.debug_temp_code_comments is in effect
        owner = c.funcstate.owner
        c.funcstate.owner = c
        expr.generate_evaluation_code(c)
        c.funcstate.owner = owner

        return expr.result()

7884 7885 7886 7887 7888
    def put_num_threads(self, code):
        """
        Write self.num_threads if set as the num_threads OpenMP directive
        """
        if self.num_threads is not None:
7889
            code.put(" num_threads(%s)" % self.evaluate_before_block(code, self.num_threads))
7890

7891

Mark Florisson's avatar
Mark Florisson committed
7892 7893 7894 7895 7896 7897 7898 7899 7900
    def declare_closure_privates(self, code):
        """
        If a variable is in a scope object, we need to allocate a temp and
        assign the value from the temp to the variable in the scope object
        after the parallel section. This kind of copying should be done only
        in the outermost parallel section.
        """
        self.modified_entries = []

7901
        for entry in sorted(self.assignments):
Mark Florisson's avatar
Mark Florisson committed
7902
            if entry.from_closure or entry.in_closure:
7903
                self._allocate_closure_temp(code, entry)
Mark Florisson's avatar
Mark Florisson committed
7904 7905

    def release_closure_privates(self, code):
7906 7907 7908
        """
        Release any temps used for variables in scope objects. As this is the
        outermost parallel block, we don't need to delete the cnames from
7909
        self.seen_closure_vars.
7910
        """
Mark Florisson's avatar
Mark Florisson committed
7911 7912 7913 7914 7915
        for entry, original_cname in self.modified_entries:
            code.putln("%s = %s;" % (original_cname, entry.cname))
            code.funcstate.release_temp(entry.cname)
            entry.cname = original_cname

7916 7917 7918 7919 7920 7921
    def privatize_temps(self, code, exclude_temps=()):
        """
        Make any used temporaries private. Before the relevant code block
        code.start_collecting_temps() should have been called.
        """
        if self.is_parallel:
7922
            c = self.privatization_insertion_point
7923

7924
            self.temps = temps = code.funcstate.stop_collecting_temps()
7925
            privates, firstprivates = [], []
7926
            for temp, type in sorted(temps):
7927
                if type.is_pyobject or type.is_memoryviewslice:
7928 7929 7930
                    firstprivates.append(temp)
                else:
                    privates.append(temp)
7931

7932 7933 7934 7935
            if privates:
                c.put(" private(%s)" % ", ".join(privates))
            if firstprivates:
                c.put(" firstprivate(%s)" % ", ".join(firstprivates))
7936 7937 7938 7939 7940 7941 7942 7943

            if self.breaking_label_used:
                shared_vars = [Naming.parallel_why]
                if self.error_label_used:
                    shared_vars.extend(self.parallel_exc)
                    c.put(" private(%s, %s, %s)" % self.pos_info)

                c.put(" shared(%s)" % ', '.join(shared_vars))
7944

7945 7946 7947 7948
    def cleanup_temps(self, code):
        # Now clean up any memoryview slice and object temporaries
        if self.is_parallel and not self.is_nested_prange:
            code.putln("/* Clean up any temporaries */")
7949
            for temp, type in sorted(self.temps):
7950 7951 7952 7953 7954
                if type.is_memoryviewslice:
                    code.put_xdecref_memoryviewslice(temp, have_gil=False)
                elif type.is_pyobject:
                    code.put_xdecref(temp, type)
                    code.putln("%s = NULL;" % temp)
7955

7956
    def setup_parallel_control_flow_block(self, code):
7957
        """
7958 7959
        Sets up a block that surrounds the parallel block to determine
        how the parallel section was exited. Any kind of return is
7960 7961 7962
        trapped (break, continue, return, exceptions). This is the idea:

        {
7963
            int why = 0;
7964 7965 7966 7967 7968 7969 7970

            #pragma omp parallel
            {
                return # -> goto new_return_label;
                goto end_parallel;

            new_return_label:
7971
                why = 3;
7972
                goto end_parallel;
7973

7974
            end_parallel:;
7975
                #pragma omp flush(why) # we need to flush for every iteration
7976 7977
            }

7978
            if (why == 3)
7979 7980 7981 7982 7983 7984 7985 7986
                goto old_return_label;
        }
        """
        self.old_loop_labels = code.new_loop_labels()
        self.old_error_label = code.new_error_label()
        self.old_return_label = code.return_label
        code.return_label = code.new_label(name="return")

7987 7988
        code.begin_block() # parallel control flow block
        self.begin_of_parallel_control_block_point = code.insertion_point()
Mark Florisson's avatar
Mark Florisson committed
7989
        self.begin_of_parallel_control_block_point_after_decls = code.insertion_point()
7990

7991 7992
        self.undef_builtin_expect_apple_gcc_bug(code)

7993 7994 7995 7996 7997 7998 7999 8000 8001 8002 8003 8004
    def begin_parallel_block(self, code):
        """
        Each OpenMP thread in a parallel section that contains a with gil block
        must have the thread-state initialized. The call to
        PyGILState_Release() then deallocates our threadstate. If we wouldn't
        do this, each with gil block would allocate and deallocate one, thereby
        losing exception information before it can be saved before leaving the
        parallel section.
        """
        self.begin_of_parallel_block = code.insertion_point()

    def end_parallel_block(self, code):
8005 8006 8007 8008 8009 8010 8011 8012 8013
        """
        To ensure all OpenMP threads have thread states, we ensure the GIL
        in each thread (which creates a thread state if it doesn't exist),
        after which we release the GIL.
        On exit, reacquire the GIL and release the thread state.

        If compiled without OpenMP support (at the C level), then we still have
        to acquire the GIL to decref any object temporaries.
        """
8014 8015 8016 8017
        if self.error_label_used:
            begin_code = self.begin_of_parallel_block
            end_code = code

8018
            begin_code.putln("#ifdef _OPENMP")
8019 8020
            begin_code.put_ensure_gil(declare_gilstate=True)
            begin_code.putln("Py_BEGIN_ALLOW_THREADS")
8021
            begin_code.putln("#endif /* _OPENMP */")
8022

8023
            end_code.putln("#ifdef _OPENMP")
8024
            end_code.putln("Py_END_ALLOW_THREADS")
8025 8026 8027 8028 8029
            end_code.putln("#else")
            end_code.put_safe("{\n")
            end_code.put_ensure_gil()
            end_code.putln("#endif /* _OPENMP */")
            self.cleanup_temps(end_code)
8030
            end_code.put_release_ensured_gil()
8031 8032
            end_code.putln("#ifndef _OPENMP")
            end_code.put_safe("}\n")
8033
            end_code.putln("#endif /* _OPENMP */")
8034

8035 8036 8037 8038
    def trap_parallel_exit(self, code, should_flush=False):
        """
        Trap any kind of return inside a parallel construct. 'should_flush'
        indicates whether the variable should be flushed, which is needed by
8039 8040 8041 8042 8043 8044 8045 8046 8047
        prange to skip the loop. It also indicates whether we need to register
        a continue (we need this for parallel blocks, but not for prange
        loops, as it is a direct jump there).

        It uses the same mechanism as try/finally:
            1 continue
            2 break
            3 return
            4 error
8048
        """
8049
        save_lastprivates_label = code.new_label()
8050
        dont_return_label = code.new_label()
8051 8052 8053

        self.any_label_used = False
        self.breaking_label_used = False
8054
        self.error_label_used = False
8055

8056 8057 8058 8059 8060 8061
        self.parallel_private_temps = []

        all_labels = code.get_all_labels()

        # Figure this out before starting to generate any code
        for label in all_labels:
8062 8063
            if code.label_used(label):
                self.breaking_label_used = (self.breaking_label_used or
8064 8065 8066 8067 8068 8069 8070 8071 8072 8073 8074
                                            label != code.continue_label)
                self.any_label_used = True

        if self.any_label_used:
            code.put_goto(dont_return_label)

        for i, label in enumerate(all_labels):
            if not code.label_used(label):
                continue

            is_continue_label = label == code.continue_label
8075

8076
            code.put_label(label)
8077

8078 8079 8080 8081
            if not (should_flush and is_continue_label):
                if label == code.error_label:
                    self.error_label_used = True
                    self.fetch_parallel_exception(code)
8082

8083
                code.putln("%s = %d;" % (Naming.parallel_why, i + 1))
8084

8085 8086 8087 8088
            if (self.breaking_label_used and self.is_prange and not
                    is_continue_label):
                code.put_goto(save_lastprivates_label)
            else:
8089 8090
                code.put_goto(dont_return_label)

8091
        if self.any_label_used:
8092 8093 8094 8095 8096
            if self.is_prange and self.breaking_label_used:
                # Don't rely on lastprivate, save our lastprivates
                code.put_label(save_lastprivates_label)
                self.save_parallel_vars(code)

8097 8098 8099
            code.put_label(dont_return_label)

            if should_flush and self.breaking_label_used:
8100 8101
                code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_why)

8102 8103 8104 8105 8106 8107 8108 8109 8110
    def save_parallel_vars(self, code):
        """
        The following shenanigans are instated when we break, return or
        propagate errors from a prange. In this case we cannot rely on
        lastprivate() to do its job, as no iterations may have executed yet
        in the last thread, leaving the values undefined. It is most likely
        that the breaking thread has well-defined values of the lastprivate
        variables, so we keep those values.
        """
8111
        section_name = "__pyx_parallel_lastprivates%d" % self.critical_section_counter
8112 8113 8114 8115 8116 8117 8118 8119
        code.putln_openmp("#pragma omp critical(%s)" % section_name)
        ParallelStatNode.critical_section_counter += 1

        code.begin_block() # begin critical section

        c = self.begin_of_parallel_control_block_point

        temp_count = 0
8120
        for entry, (op, lastprivate) in sorted(self.privates.items()):
8121 8122 8123
            if not lastprivate or entry.type.is_pyobject:
                continue

8124
            type_decl = entry.type.empty_declaration_code()
8125 8126 8127 8128 8129
            temp_cname = "__pyx_parallel_temp%d" % temp_count
            private_cname = entry.cname

            temp_count += 1

8130 8131
            invalid_value = entry.type.invalid_value()
            if invalid_value:
8132
                init = ' = ' + entry.type.cast_code(invalid_value)
8133 8134
            else:
                init = ''
8135
            # Declare the parallel private in the outer block
8136
            c.putln("%s %s%s;" % (type_decl, temp_cname, init))
8137 8138 8139 8140 8141 8142 8143 8144

            # Initialize before escaping
            code.putln("%s = %s;" % (temp_cname, private_cname))

            self.parallel_private_temps.append((temp_cname, private_cname))

        code.end_block() # end critical section

8145 8146 8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178
    def fetch_parallel_exception(self, code):
        """
        As each OpenMP thread may raise an exception, we need to fetch that
        exception from the threadstate and save it for after the parallel
        section where it can be re-raised in the master thread.

        Although it would seem that __pyx_filename, __pyx_lineno and
        __pyx_clineno are only assigned to under exception conditions (i.e.,
        when we have the GIL), and thus should be allowed to be shared without
        any race condition, they are in fact subject to the same race
        conditions that they were previously when they were global variables
        and functions were allowed to release the GIL:

            thread A                thread B
                acquire
                set lineno
                release
                                        acquire
                                        set lineno
                                        release
                acquire
                fetch exception
                release
                                        skip the fetch

                deallocate threadstate  deallocate threadstate
        """
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

        code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_exc_type)
        code.putln(
            "if (!%s) {" % Naming.parallel_exc_type)

8179
        code.putln("__Pyx_ErrFetchWithState(&%s, &%s, &%s);" % self.parallel_exc)
8180
        pos_info = chain(*zip(self.parallel_pos_info, self.pos_info))
8181
        code.funcstate.uses_error_indicator = True
8182
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
8183
        code.put_gotref(Naming.parallel_exc_type)
8184 8185 8186 8187 8188 8189 8190 8191 8192 8193 8194 8195

        code.putln(
            "}")

        code.put_release_ensured_gil()
        code.end_block()

    def restore_parallel_exception(self, code):
        "Re-raise a parallel exception"
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

8196
        code.put_giveref(Naming.parallel_exc_type)
8197
        code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc)
8198 8199 8200 8201 8202
        pos_info = chain(*zip(self.pos_info, self.parallel_pos_info))
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))

        code.put_release_ensured_gil()
        code.end_block()
8203 8204

    def restore_labels(self, code):
8205 8206 8207 8208 8209 8210
        """
        Restore all old labels. Call this before the 'else' clause to for
        loops and always before ending the parallel control flow block.
        """
        code.set_all_labels(self.old_loop_labels + (self.old_return_label,
                                                    self.old_error_label))
8211

8212 8213
    def end_parallel_control_flow_block(
            self, code, break_=False, continue_=False, return_=False):
8214 8215 8216 8217 8218
        """
        This ends the parallel control flow block and based on how the parallel
        section was exited, takes the corresponding action. The break_ and
        continue_ parameters indicate whether these should be propagated
        outwards:
8219

8220 8221 8222 8223 8224 8225 8226
            for i in prange(...):
                with cython.parallel.parallel():
                    continue

        Here break should be trapped in the parallel block, and propagated to
        the for loop.
        """
8227 8228 8229 8230
        c = self.begin_of_parallel_control_block_point

        # Firstly, always prefer errors over returning, continue or break
        if self.error_label_used:
8231 8232
            c.putln("const char *%s = NULL; int %s = 0, %s = 0;" % self.parallel_pos_info)
            c.putln("PyObject *%s = NULL, *%s = NULL, *%s = NULL;" % self.parallel_exc)
8233 8234 8235 8236 8237 8238 8239 8240 8241

            code.putln(
                "if (%s) {" % Naming.parallel_exc_type)
            code.putln("/* This may have been overridden by a continue, "
                       "break or return in another thread. Prefer the error. */")
            code.putln("%s = 4;" % Naming.parallel_why)
            code.putln(
                "}")

8242 8243 8244 8245 8246 8247 8248
        if continue_:
            any_label_used = self.any_label_used
        else:
            any_label_used = self.breaking_label_used

        if any_label_used:
            # __pyx_parallel_why is used, declare and initialize
8249 8250
            c.putln("int %s;" % Naming.parallel_why)
            c.putln("%s = 0;" % Naming.parallel_why)
8251

8252 8253 8254 8255 8256 8257
            code.putln(
                "if (%s) {" % Naming.parallel_why)

            for temp_cname, private_cname in self.parallel_private_temps:
                code.putln("%s = %s;" % (private_cname, temp_cname))

8258
            code.putln("switch (%s) {" % Naming.parallel_why)
8259 8260 8261 8262 8263 8264 8265 8266
            if continue_:
                code.put("    case 1: ")
                code.put_goto(code.continue_label)

            if break_:
                code.put("    case 2: ")
                code.put_goto(code.break_label)

8267 8268 8269
            if return_:
                code.put("    case 3: ")
                code.put_goto(code.return_label)
8270 8271

            if self.error_label_used:
8272
                code.globalstate.use_utility_code(restore_exception_utility_code)
8273 8274 8275
                code.putln("    case 4:")
                self.restore_parallel_exception(code)
                code.put_goto(code.error_label)
8276

8277 8278 8279
            code.putln("}") # end switch
            code.putln(
                "}") # end if
8280 8281

        code.end_block() # end parallel control flow block
8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293
        self.redef_builtin_expect_apple_gcc_bug(code)

    # FIXME: improve with version number for OS X Lion
    buggy_platform_macro_condition = "(defined(__APPLE__) || defined(__OSX__))"
    have_expect_condition = "(defined(__GNUC__) && " \
                             "(__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))))"
    redef_condition = "(%s && %s)" % (buggy_platform_macro_condition, have_expect_condition)

    def undef_builtin_expect_apple_gcc_bug(self, code):
        """
        A bug on OS X Lion disallows __builtin_expect macros. This code avoids them
        """
8294
        if not self.parent:
8295
            code.undef_builtin_expect(self.redef_condition)
8296 8297

    def redef_builtin_expect_apple_gcc_bug(self, code):
8298
        if not self.parent:
8299
            code.redef_builtin_expect(self.redef_condition)
8300

Mark Florisson's avatar
Mark Florisson committed
8301 8302 8303

class ParallelWithBlockNode(ParallelStatNode):
    """
8304
    This node represents a 'with cython.parallel.parallel():' block
Mark Florisson's avatar
Mark Florisson committed
8305 8306
    """

8307 8308 8309 8310 8311 8312 8313 8314 8315
    valid_keyword_arguments = ['num_threads']

    num_threads = None

    def analyse_declarations(self, env):
        super(ParallelWithBlockNode, self).analyse_declarations(env)
        if self.args:
            error(self.pos, "cython.parallel.parallel() does not take "
                            "positional arguments")
8316

Mark Florisson's avatar
Mark Florisson committed
8317 8318
    def generate_execution_code(self, code):
        self.declare_closure_privates(code)
8319
        self.setup_parallel_control_flow_block(code)
Mark Florisson's avatar
Mark Florisson committed
8320 8321 8322

        code.putln("#ifdef _OPENMP")
        code.put("#pragma omp parallel ")
8323 8324

        if self.privates:
8325
            privates = [e.cname for e in self.privates
8326
                        if not e.type.is_pyobject]
8327
            code.put('private(%s)' % ', '.join(sorted(privates)))
8328

8329
        self.privatization_insertion_point = code.insertion_point()
8330
        self.put_num_threads(code)
8331
        code.putln("")
8332

8333 8334
        code.putln("#endif /* _OPENMP */")

8335
        code.begin_block()  # parallel block
8336
        self.begin_parallel_block(code)
8337
        self.initialize_privates_to_nan(code)
8338
        code.funcstate.start_collecting_temps()
Mark Florisson's avatar
Mark Florisson committed
8339
        self.body.generate_execution_code(code)
8340
        self.trap_parallel_exit(code)
8341
        self.privatize_temps(code)
8342
        self.end_parallel_block(code)
8343
        code.end_block()  # end parallel block
8344

8345 8346
        continue_ = code.label_used(code.continue_label)
        break_ = code.label_used(code.break_label)
8347
        return_ = code.label_used(code.return_label)
Mark Florisson's avatar
Mark Florisson committed
8348

8349 8350
        self.restore_labels(code)
        self.end_parallel_control_flow_block(code, break_=break_,
8351 8352
                                             continue_=continue_,
                                             return_=return_)
Mark Florisson's avatar
Mark Florisson committed
8353 8354 8355 8356 8357 8358 8359 8360 8361 8362 8363
        self.release_closure_privates(code)


class ParallelRangeNode(ParallelStatNode):
    """
    This node represents a 'for i in cython.parallel.prange():' construct.

    target       NameNode       the target iteration variable
    else_clause  Node or None   the else clause of this loop
    """

8364 8365
    child_attrs = ['body', 'target', 'else_clause', 'args', 'num_threads',
                   'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8366 8367 8368 8369 8370 8371

    body = target = else_clause = args = None

    start = stop = step = None

    is_prange = True
8372

8373
    nogil = None
8374 8375
    schedule = None

8376
    valid_keyword_arguments = ['schedule', 'nogil', 'num_threads', 'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8377

8378 8379 8380 8381 8382
    def __init__(self, pos, **kwds):
        super(ParallelRangeNode, self).__init__(pos, **kwds)
        # Pretend to be a ForInStatNode for control flow analysis
        self.iterator = PassStatNode(pos)

Mark Florisson's avatar
Mark Florisson committed
8383 8384 8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399
    def analyse_declarations(self, env):
        super(ParallelRangeNode, self).analyse_declarations(env)
        self.target.analyse_target_declaration(env)
        if self.else_clause is not None:
            self.else_clause.analyse_declarations(env)

        if not self.args or len(self.args) > 3:
            error(self.pos, "Invalid number of positional arguments to prange")
            return

        if len(self.args) == 1:
            self.stop, = self.args
        elif len(self.args) == 2:
            self.start, self.stop = self.args
        else:
            self.start, self.stop, self.step = self.args

Mark Florisson's avatar
Mark Florisson committed
8400 8401 8402
        if hasattr(self.schedule, 'decode'):
            self.schedule = self.schedule.decode('ascii')

8403 8404
        if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'):
            error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,))
Mark Florisson's avatar
Mark Florisson committed
8405 8406

    def analyse_expressions(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
8407
        was_nogil = env.nogil
8408 8409 8410
        if self.nogil:
            env.nogil = True

8411 8412
        if self.target is None:
            error(self.pos, "prange() can only be used as part of a for loop")
8413
            return self
Mark Florisson's avatar
Mark Florisson committed
8414

8415
        self.target = self.target.analyse_target_types(env)
Mark Florisson's avatar
Mark Florisson committed
8416

8417 8418 8419 8420 8421 8422 8423 8424
        if not self.target.type.is_numeric:
            # Not a valid type, assume one for now anyway

            if not self.target.type.is_pyobject:
                # nogil_check will catch the is_pyobject case
                error(self.target.pos,
                      "Must be of numeric type, not %s" % self.target.type)

8425
            self.index_type = PyrexTypes.c_py_ssize_t_type
8426 8427
        else:
            self.index_type = self.target.type
8428 8429 8430 8431
            if not self.index_type.signed:
                warning(self.target.pos,
                        "Unsigned index type not allowed before OpenMP 3.0",
                        level=2)
Mark Florisson's avatar
Mark Florisson committed
8432 8433 8434 8435 8436 8437 8438 8439 8440

        # Setup start, stop and step, allocating temps if needed
        self.names = 'start', 'stop', 'step'
        start_stop_step = self.start, self.stop, self.step

        for node, name in zip(start_stop_step, self.names):
            if node is not None:
                node.analyse_types(env)
                if not node.type.is_numeric:
8441 8442
                    error(node.pos, "%s argument must be numeric" % name)
                    continue
Mark Florisson's avatar
Mark Florisson committed
8443 8444 8445 8446 8447 8448 8449 8450

                if not node.is_literal:
                    node = node.coerce_to_temp(env)
                    setattr(self, name, node)

                # As we range from 0 to nsteps, computing the index along the
                # way, we need a fitting type for 'i' and 'nsteps'
                self.index_type = PyrexTypes.widest_numeric_type(
8451
                    self.index_type, node.type)
Mark Florisson's avatar
Mark Florisson committed
8452 8453

        if self.else_clause is not None:
8454
            self.else_clause = self.else_clause.analyse_expressions(env)
Mark Florisson's avatar
Mark Florisson committed
8455

8456 8457 8458 8459 8460 8461 8462 8463
        # Although not actually an assignment in this scope, it should be
        # treated as such to ensure it is unpacked if a closure temp, and to
        # ensure lastprivate behaviour and propagation. If the target index is
        # not a NameNode, it won't have an entry, and an error was issued by
        # ParallelRangeTransform
        if hasattr(self.target, 'entry'):
            self.assignments[self.target.entry] = self.target.pos, None

8464
        node = super(ParallelRangeNode, self).analyse_expressions(env)
8465

8466 8467 8468
        if node.chunksize:
            if not node.schedule:
                error(node.chunksize.pos,
8469
                      "Must provide schedule with chunksize")
8470 8471
            elif node.schedule == 'runtime':
                error(node.chunksize.pos,
8472
                      "Chunksize not valid for the schedule runtime")
8473 8474 8475 8476
            elif (node.chunksize.type.is_int and
                  node.chunksize.is_literal and
                  node.chunksize.compile_time_value(env) <= 0):
                error(node.chunksize.pos, "Chunksize must not be negative")
8477

8478 8479
            node.chunksize = node.chunksize.coerce_to(
                PyrexTypes.c_int_type, env).coerce_to_temp(env)
8480

8481
        if node.nogil:
8482 8483
            env.nogil = was_nogil

8484 8485 8486
        node.is_nested_prange = node.parent and node.parent.is_prange
        if node.is_nested_prange:
            parent = node
8487 8488 8489
            while parent.parent and parent.parent.is_prange:
                parent = parent.parent

8490 8491 8492 8493
            parent.assignments.update(node.assignments)
            parent.privates.update(node.privates)
            parent.assigned_nodes.extend(node.assigned_nodes)
        return node
8494

Mark Florisson's avatar
Mark Florisson committed
8495 8496
    def nogil_check(self, env):
        names = 'start', 'stop', 'step', 'target'
8497
        nodes = self.start, self.stop, self.step, self.target
Mark Florisson's avatar
Mark Florisson committed
8498 8499 8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542
        for name, node in zip(names, nodes):
            if node is not None and node.type.is_pyobject:
                error(node.pos, "%s may not be a Python object "
                                "as we don't have the GIL" % name)

    def generate_execution_code(self, code):
        """
        Generate code in the following steps

            1)  copy any closure variables determined thread-private
                into temporaries

            2)  allocate temps for start, stop and step

            3)  generate a loop that calculates the total number of steps,
                which then computes the target iteration variable for every step:

                    for i in prange(start, stop, step):
                        ...

                becomes

                    nsteps = (stop - start) / step;
                    i = start;

                    #pragma omp parallel for lastprivate(i)
                    for (temp = 0; temp < nsteps; temp++) {
                        i = start + step * temp;
                        ...
                    }

                Note that accumulation of 'i' would have a data dependency
                between iterations.

                Also, you can't do this

                    for (i = start; i < stop; i += step)
                        ...

                as the '<' operator should become '>' for descending loops.
                'for i from x < i < y:' does not suffer from this problem
                as the relational operator is known at compile time!

            4) release our temps and write back any private closure variables
        """
8543
        self.declare_closure_privates(code)
Mark Florisson's avatar
Mark Florisson committed
8544 8545 8546 8547 8548 8549 8550 8551

        # This can only be a NameNode
        target_index_cname = self.target.entry.cname

        # This will be used as the dict to format our code strings, holding
        # the start, stop , step, temps and target cnames
        fmt_dict = {
            'target': target_index_cname,
8552
            'target_type': self.target.type.empty_declaration_code()
Mark Florisson's avatar
Mark Florisson committed
8553 8554 8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575
        }

        # Setup start, stop and step, allocating temps if needed
        start_stop_step = self.start, self.stop, self.step
        defaults = '0', '0', '1'
        for node, name, default in zip(start_stop_step, self.names, defaults):
            if node is None:
                result = default
            elif node.is_literal:
                result = node.get_constant_c_result_code()
            else:
                node.generate_evaluation_code(code)
                result = node.result()

            fmt_dict[name] = result

        fmt_dict['i'] = code.funcstate.allocate_temp(self.index_type, False)
        fmt_dict['nsteps'] = code.funcstate.allocate_temp(self.index_type, False)

        # TODO: check if the step is 0 and if so, raise an exception in a
        # 'with gil' block. For now, just abort
        code.putln("if (%(step)s == 0) abort();" % fmt_dict)

8576
        self.setup_parallel_control_flow_block(code) # parallel control flow block
8577

8578
        self.control_flow_var_code_point = code.insertion_point()
8579

8580
        # Note: nsteps is private in an outer scope if present
8581
        code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict)
Mark Florisson's avatar
Mark Florisson committed
8582

8583 8584 8585 8586 8587 8588 8589
        # The target iteration variable might not be initialized, do it only if
        # we are executing at least 1 iteration, otherwise we should leave the
        # target unaffected. The target iteration variable is firstprivate to
        # shut up compiler warnings caused by lastprivate, as the compiler
        # erroneously believes that nsteps may be <= 0, leaving the private
        # target index uninitialized
        code.putln("if (%(nsteps)s > 0)" % fmt_dict)
8590
        code.begin_block() # if block
Mark Florisson's avatar
Mark Florisson committed
8591
        self.generate_loop(code, fmt_dict)
8592
        code.end_block() # end if block
Mark Florisson's avatar
Mark Florisson committed
8593

8594 8595 8596
        self.restore_labels(code)

        if self.else_clause:
8597
            if self.breaking_label_used:
8598
                code.put("if (%s < 2)" % Naming.parallel_why)
8599 8600 8601 8602 8603 8604 8605

            code.begin_block() # else block
            code.putln("/* else */")
            self.else_clause.generate_execution_code(code)
            code.end_block() # end else block

        # ------ cleanup ------
8606
        self.end_parallel_control_flow_block(code) # end parallel control flow block
8607

Mark Florisson's avatar
Mark Florisson committed
8608 8609
        # And finally, release our privates and write back any closure
        # variables
8610
        for temp in start_stop_step + (self.chunksize, self.num_threads):
Mark Florisson's avatar
Mark Florisson committed
8611 8612 8613 8614 8615 8616 8617 8618 8619 8620
            if temp is not None:
                temp.generate_disposal_code(code)
                temp.free_temps(code)

        code.funcstate.release_temp(fmt_dict['i'])
        code.funcstate.release_temp(fmt_dict['nsteps'])

        self.release_closure_privates(code)

    def generate_loop(self, code, fmt_dict):
8621 8622 8623 8624
        if self.is_nested_prange:
            code.putln("#if 0")
        else:
            code.putln("#ifdef _OPENMP")
Mark Florisson's avatar
Mark Florisson committed
8625 8626 8627

        if not self.is_parallel:
            code.put("#pragma omp for")
8628
            self.privatization_insertion_point = code.insertion_point()
8629
            reduction_codepoint = self.parent.privatization_insertion_point
Mark Florisson's avatar
Mark Florisson committed
8630
        else:
8631 8632
            code.put("#pragma omp parallel")
            self.privatization_insertion_point = code.insertion_point()
8633
            reduction_codepoint = self.privatization_insertion_point
8634 8635 8636 8637 8638 8639 8640 8641
            code.putln("")
            code.putln("#endif /* _OPENMP */")

            code.begin_block() # pragma omp parallel begin block

            # Initialize the GIL if needed for this thread
            self.begin_parallel_block(code)

8642 8643 8644 8645
            if self.is_nested_prange:
                code.putln("#if 0")
            else:
                code.putln("#ifdef _OPENMP")
8646
            code.put("#pragma omp for")
Mark Florisson's avatar
Mark Florisson committed
8647

8648
        for entry, (op, lastprivate) in sorted(self.privates.items()):
Mark Florisson's avatar
Mark Florisson committed
8649
            # Don't declare the index variable as a reduction
8650
            if op and op in "+*-&^|" and entry != self.target.entry:
8651 8652 8653
                if entry.type.is_pyobject:
                    error(self.pos, "Python objects cannot be reductions")
                else:
8654 8655 8656 8657
                    #code.put(" reduction(%s:%s)" % (op, entry.cname))
                    # This is the only way reductions + nesting works in gcc4.5
                    reduction_codepoint.put(
                                " reduction(%s:%s)" % (op, entry.cname))
8658
            else:
8659 8660
                if entry == self.target.entry:
                    code.put(" firstprivate(%s)" % entry.cname)
8661 8662
                    code.put(" lastprivate(%s)" % entry.cname)
                    continue
Mark Florisson's avatar
Mark Florisson committed
8663

8664
                if not entry.type.is_pyobject:
8665 8666 8667 8668
                    if lastprivate:
                        private = 'lastprivate'
                    else:
                        private = 'private'
Mark Florisson's avatar
Mark Florisson committed
8669

8670
                    code.put(" %s(%s)" % (private, entry.cname))
8671

Mark Florisson's avatar
Mark Florisson committed
8672
        if self.schedule:
8673
            if self.chunksize:
8674
                chunksize = ", %s" % self.evaluate_before_block(code, self.chunksize)
8675 8676 8677 8678
            else:
                chunksize = ""

            code.put(" schedule(%s%s)" % (self.schedule, chunksize))
8679

8680
        self.put_num_threads(reduction_codepoint)
8681

8682 8683
        code.putln("")
        code.putln("#endif /* _OPENMP */")
Mark Florisson's avatar
Mark Florisson committed
8684 8685

        code.put("for (%(i)s = 0; %(i)s < %(nsteps)s; %(i)s++)" % fmt_dict)
8686
        code.begin_block()  # for loop block
8687

8688
        guard_around_body_codepoint = code.insertion_point()
8689

8690 8691
        # Start if guard block around the body. This may be unnecessary, but
        # at least it doesn't spoil indentation
Mark Florisson's avatar
Mark Florisson committed
8692
        code.begin_block()
8693

8694
        code.putln("%(target)s = (%(target_type)s)(%(start)s + %(step)s * %(i)s);" % fmt_dict)
8695 8696
        self.initialize_privates_to_nan(code, exclude=self.target.entry)

8697 8698 8699
        if self.is_parallel:
            code.funcstate.start_collecting_temps()

Mark Florisson's avatar
Mark Florisson committed
8700
        self.body.generate_execution_code(code)
8701
        self.trap_parallel_exit(code, should_flush=True)
8702 8703
        self.privatize_temps(code)

8704 8705 8706
        if self.breaking_label_used:
            # Put a guard around the loop body in case return, break or
            # exceptions might be used
8707
            guard_around_body_codepoint.putln("if (%s < 2)" % Naming.parallel_why)
8708

8709 8710
        code.end_block()  # end guard around loop body
        code.end_block()  # end for loop block
Mark Florisson's avatar
Mark Florisson committed
8711

8712 8713 8714
        if self.is_parallel:
            # Release the GIL and deallocate the thread state
            self.end_parallel_block(code)
8715
            code.end_block()  # pragma omp parallel end block
8716

8717

8718 8719 8720 8721 8722 8723 8724 8725
class CnameDecoratorNode(StatNode):
    """
    This node is for the cname decorator in CythonUtilityCode:

        @cname('the_cname')
        cdef func(...):
            ...

8726 8727
    In case of a cdef class the cname specifies the objstruct_cname.

8728 8729 8730 8731 8732 8733 8734 8735
    node        the node to which the cname decorator is applied
    cname       the cname the node should get
    """

    child_attrs = ['node']

    def analyse_declarations(self, env):
        self.node.analyse_declarations(env)
8736

8737 8738 8739 8740 8741
        node = self.node
        if isinstance(node, CompilerDirectivesNode):
            node = node.body.stats[0]

        self.is_function = isinstance(node, FuncDefNode)
8742
        is_struct_or_enum = isinstance(node, (CStructOrUnionDefNode, CEnumDefNode))
8743
        e = node.entry
8744 8745 8746 8747

        if self.is_function:
            e.cname = self.cname
            e.func_cname = self.cname
8748
            e.used = True
8749 8750
            if e.pyfunc_cname and '.' in e.pyfunc_cname:
                e.pyfunc_cname = self.mangle(e.pyfunc_cname)
8751 8752
        elif is_struct_or_enum:
            e.cname = e.type.cname = self.cname
8753
        else:
8754
            scope = node.scope
8755 8756

            e.cname = self.cname
8757
            e.type.objstruct_cname = self.cname + '_obj'
8758
            e.type.typeobj_cname = Naming.typeobj_prefix + self.cname
8759
            e.type.typeptr_cname = self.cname + '_type'
8760
            e.type.scope.namespace_cname = e.type.typeptr_cname
8761

8762
            e.as_variable.cname = e.type.typeptr_cname
8763 8764 8765

            scope.scope_prefix = self.cname + "_"

8766
            for name, entry in scope.entries.items():
8767
                if entry.func_cname:
8768 8769 8770 8771 8772 8773 8774 8775 8776
                    entry.func_cname = self.mangle(entry.cname)
                if entry.pyfunc_cname:
                    entry.pyfunc_cname = self.mangle(entry.pyfunc_cname)

    def mangle(self, cname):
        if '.' in cname:
            # remove __pyx_base from func_cname
            cname = cname.split('.')[-1]
        return '%s_%s' % (self.cname, cname)
8777 8778

    def analyse_expressions(self, env):
8779 8780
        self.node = self.node.analyse_expressions(env)
        return self
8781 8782

    def generate_function_definitions(self, env, code):
8783
        "Ensure a prototype for every @cname method in the right place"
8784 8785 8786 8787 8788 8789
        if self.is_function and env.is_c_class_scope:
            # method in cdef class, generate a prototype in the header
            h_code = code.globalstate['utility_code_proto']

            if isinstance(self.node, DefNode):
                self.node.generate_function_header(
8790
                    h_code, with_pymethdef=False, proto_only=True)
8791
            else:
8792
                from . import ModuleNode
8793 8794 8795 8796 8797
                entry = self.node.entry
                cname = entry.cname
                entry.cname = entry.func_cname

                ModuleNode.generate_cfunction_declaration(
8798 8799 8800 8801
                    entry,
                    env.global_scope(),
                    h_code,
                    definition=True)
8802 8803 8804 8805 8806 8807 8808

                entry.cname = cname

        self.node.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
        self.node.generate_execution_code(code)
Mark Florisson's avatar
Mark Florisson committed
8809

8810

William Stein's avatar
William Stein committed
8811 8812 8813 8814 8815 8816
#------------------------------------------------------------------------------------
#
#  Runtime support code
#
#------------------------------------------------------------------------------------

Robert Bradshaw's avatar
Robert Bradshaw committed
8817
if Options.gcc_branch_hints:
Stefan Behnel's avatar
Stefan Behnel committed
8818
    branch_prediction_macros = """
8819 8820 8821 8822 8823 8824
/* Test for GCC > 2.95 */
#if defined(__GNUC__) \
    && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
  #define likely(x)   __builtin_expect(!!(x), 1)
  #define unlikely(x) __builtin_expect(!!(x), 0)
#else /* !__GNUC__ or GCC < 2.95 */
8825 8826
  #define likely(x)   (x)
  #define unlikely(x) (x)
8827
#endif /* __GNUC__ */
Stefan Behnel's avatar
Stefan Behnel committed
8828
"""
Robert Bradshaw's avatar
Robert Bradshaw committed
8829
else:
Stefan Behnel's avatar
Stefan Behnel committed
8830
    branch_prediction_macros = """
Robert Bradshaw's avatar
Robert Bradshaw committed
8831 8832
#define likely(x)   (x)
#define unlikely(x) (x)
Stefan Behnel's avatar
Stefan Behnel committed
8833
"""
William Stein's avatar
William Stein committed
8834 8835 8836

#------------------------------------------------------------------------------------

8837 8838
printing_utility_code = UtilityCode.load_cached("Print", "Printing.c")
printing_one_utility_code = UtilityCode.load_cached("PrintOne", "Printing.c")
8839

William Stein's avatar
William Stein committed
8840 8841
#------------------------------------------------------------------------------------

8842 8843 8844 8845 8846 8847 8848
# Exception raising code
#
# Exceptions are raised by __Pyx_Raise() and stored as plain
# type/value/tb in PyThreadState->curexc_*.  When being caught by an
# 'except' statement, curexc_* is moved over to exc_* by
# __Pyx_GetException()

8849 8850 8851 8852 8853
restore_exception_utility_code = UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")
raise_utility_code = UtilityCode.load_cached("RaiseException", "Exceptions.c")
get_exception_utility_code = UtilityCode.load_cached("GetException", "Exceptions.c")
swap_exception_utility_code = UtilityCode.load_cached("SwapException", "Exceptions.c")
reset_exception_utility_code = UtilityCode.load_cached("SaveResetException", "Exceptions.c")
8854
traceback_utility_code = UtilityCode.load_cached("AddTraceback", "Exceptions.c")
8855 8856 8857

#------------------------------------------------------------------------------------

8858 8859
get_exception_tuple_utility_code = UtilityCode(
    proto="""
8860
static PyObject *__Pyx_GetExceptionTuple(PyThreadState *__pyx_tstate); /*proto*/
8861
""",
8862 8863 8864 8865 8866
    # I doubt that calling __Pyx_GetException() here is correct as it moves
    # the exception from tstate->curexc_* to tstate->exc_*, which prevents
    # exception handlers later on from receiving it.
    # NOTE: "__pyx_tstate" may be used by __Pyx_GetException() macro
    impl = """
8867
static PyObject *__Pyx_GetExceptionTuple(CYTHON_UNUSED PyThreadState *__pyx_tstate) {
8868 8869 8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883
    PyObject *type = NULL, *value = NULL, *tb = NULL;
    if (__Pyx_GetException(&type, &value, &tb) == 0) {
        PyObject* exc_info = PyTuple_New(3);
        if (exc_info) {
            Py_INCREF(type);
            Py_INCREF(value);
            Py_INCREF(tb);
            PyTuple_SET_ITEM(exc_info, 0, type);
            PyTuple_SET_ITEM(exc_info, 1, value);
            PyTuple_SET_ITEM(exc_info, 2, tb);
            return exc_info;
        }
    }
    return NULL;
}
""",
8884
    requires=[get_exception_utility_code])