Nodes.py 353 KB
Newer Older
William Stein's avatar
William Stein committed
1
#
2
#   Parse tree nodes
William Stein's avatar
William Stein committed
3
#
4

5 6
from __future__ import absolute_import

7
import cython
8
cython.declare(sys=object, os=object, copy=object,
9
               Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object,
10
               py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object,
11
               StructOrUnionScope=object, PyClassScope=object,
12
               CppClassScope=object, UtilityCode=object, EncodedString=object,
13
               error_type=object, _py_int_types=object)
William Stein's avatar
William Stein committed
14

15
import sys, os, copy
Stefan Behnel's avatar
Stefan Behnel committed
16
from itertools import chain
Robert Bradshaw's avatar
Robert Bradshaw committed
17

18 19 20 21 22 23 24
from . import Builtin
from .Errors import error, warning, InternalError, CompileError
from . import Naming
from . import PyrexTypes
from . import TypeSlots
from .PyrexTypes import py_object_type, error_type
from .Symtab import (ModuleScope, LocalScope, ClosureScope,
25
                     StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope)
26
from .Code import UtilityCode
27
from .StringEncoding import EncodedString
28
from . import Future
29 30
from . import Options
from . import DebugFlags
31
from .Pythran import has_np_pythran, pythran_type, is_pythran_buffer
32
from ..Utils import add_metaclass
William Stein's avatar
William Stein committed
33

34

35 36 37 38 39
if sys.version_info[0] >= 3:
    _py_int_types = int
else:
    _py_int_types = (int, long)

40

41
def relative_position(pos):
42
    return (pos[0].get_filenametable_entry(), pos[1])
43

44 45 46 47

def embed_position(pos, docstring):
    if not Options.embed_pos_in_docstring:
        return docstring
48
    pos_line = u'File: %s (starting at line %s)' % relative_position(pos)
49 50
    if docstring is None:
        # unicode string
51
        return EncodedString(pos_line)
52 53 54 55 56 57

    # make sure we can encode the filename in the docstring encoding
    # otherwise make the docstring a unicode string
    encoding = docstring.encoding
    if encoding is not None:
        try:
Stefan Behnel's avatar
Stefan Behnel committed
58
            pos_line.encode(encoding)
59 60 61 62 63
        except UnicodeEncodeError:
            encoding = None

    if not docstring:
        # reuse the string encoding of the original docstring
64
        doc = EncodedString(pos_line)
65
    else:
66
        doc = EncodedString(pos_line + u'\n' + docstring)
67 68
    doc.encoding = encoding
    return doc
69

70

71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98
def _analyse_signature_annotation(annotation, env):
    base_type = None
    explicit_pytype = explicit_ctype = False
    if annotation.is_dict_literal:
        for name, value in annotation.key_value_pairs:
            if not name.is_string_literal:
                continue
            if name.value in ('type', b'type'):
                explicit_pytype = True
                if not explicit_ctype:
                    annotation = value
            elif name.value in ('ctype', b'ctype'):
                explicit_ctype = True
                annotation = value
        if explicit_pytype and explicit_ctype:
            warning(annotation.pos, "Duplicate type declarations found in signature annotation")
    arg_type = annotation.analyse_as_type(env)
    if arg_type is not None:
        if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject:
            warning(annotation.pos,
                    "Python type declaration in signature annotation does not refer to a Python type")
        base_type = CAnalysedBaseTypeNode(
            annotation.pos, type=arg_type, is_arg=True)
    else:
        warning(annotation.pos, "Unknown type declaration found in signature annotation")
    return base_type, arg_type


99
def write_func_call(func, codewriter_class):
100
    def f(*args, **kwds):
101
        if len(args) > 1 and isinstance(args[1], codewriter_class):
Robert Bradshaw's avatar
Robert Bradshaw committed
102 103
            # here we annotate the code with this function call
            # but only if new code is generated
104
            node, code = args[:2]
Robert Bradshaw's avatar
Robert Bradshaw committed
105
            marker = '                    /* %s -> %s.%s %s */' % (
Stefan Behnel's avatar
Stefan Behnel committed
106 107 108 109
                ' ' * code.call_level,
                node.__class__.__name__,
                func.__name__,
                node.pos[1:])
Robert Bradshaw's avatar
Robert Bradshaw committed
110 111 112 113 114 115 116
            pristine = code.buffer.stream.tell()
            code.putln(marker)
            start = code.buffer.stream.tell()
            code.call_level += 4
            res = func(*args, **kwds)
            code.call_level -= 4
            if start == code.buffer.stream.tell():
117 118
                # no code written => undo writing marker
                code.buffer.stream.truncate(pristine)
Robert Bradshaw's avatar
Robert Bradshaw committed
119
            else:
120
                marker = marker.replace('->', '<-', 1)
Robert Bradshaw's avatar
Robert Bradshaw committed
121 122 123 124
                code.putln(marker)
            return res
        else:
            return func(*args, **kwds)
125 126
    return f

127

128 129
class VerboseCodeWriter(type):
    # Set this as a metaclass to trace function calls in code.
130
    # This slows down code generation and makes much larger files.
131
    def __new__(cls, name, bases, attrs):
132
        from types import FunctionType
133
        from .Code import CCodeWriter
134 135 136
        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType):
137
                attrs[mname] = write_func_call(m, CCodeWriter)
138 139 140
        return super(VerboseCodeWriter, cls).__new__(cls, name, bases, attrs)


141 142 143 144 145 146 147 148 149 150 151 152 153
class CheckAnalysers(type):
    """Metaclass to check that type analysis functions return a node.
    """
    methods = set(['analyse_types',
                   'analyse_expressions',
                   'analyse_target_types'])

    def __new__(cls, name, bases, attrs):
        from types import FunctionType
        def check(name, func):
            def call(*args, **kwargs):
                retval = func(*args, **kwargs)
                if retval is None:
154
                    print('%s %s %s' % (name, args, kwargs))
155 156 157 158 159 160 161 162 163 164
                return retval
            return call

        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType) and mname in cls.methods:
                attrs[mname] = check(mname, m)
        return super(CheckAnalysers, cls).__new__(cls, name, bases, attrs)


165 166 167 168 169 170 171 172
def _with_metaclass(cls):
    if DebugFlags.debug_trace_code_generation:
        return add_metaclass(VerboseCodeWriter)(cls)
    #return add_metaclass(CheckAnalysers)(cls)
    return cls


@_with_metaclass
Stefan Behnel's avatar
Stefan Behnel committed
173
class Node(object):
William Stein's avatar
William Stein committed
174 175 176
    #  pos         (string, int, int)   Source file position
    #  is_name     boolean              Is a NameNode
    #  is_literal  boolean              Is a ConstNode
177

William Stein's avatar
William Stein committed
178
    is_name = 0
179
    is_none = 0
180
    is_nonecheck = 0
William Stein's avatar
William Stein committed
181
    is_literal = 0
182
    is_terminator = 0
183
    is_wrapper = False  # is a DefNode wrapper for a C function
184
    temps = None
185

Stefan Behnel's avatar
Stefan Behnel committed
186
    # All descendants should set child_attrs to a list of the attributes
187 188
    # containing nodes considered "children" in the tree. Each such attribute
    # can either contain a single node or a list of nodes. See Visitor.py.
189
    child_attrs = None
190

191 192
    cf_state = None

193 194 195 196 197
    # This may be an additional (or 'actual') type that will be checked when
    # this node is coerced to another type. This could be useful to set when
    # the actual type to which it can coerce is known, but you want to leave
    # the type a py_object_type
    coercion_type = None
198

William Stein's avatar
William Stein committed
199 200 201
    def __init__(self, pos, **kw):
        self.pos = pos
        self.__dict__.update(kw)
202

203 204
    gil_message = "Operation"

205
    nogil_check = None
206

207
    def gil_error(self, env=None):
208
        error(self.pos, "%s not allowed without gil" % self.gil_message)
209

Robert Bradshaw's avatar
Robert Bradshaw committed
210
    cpp_message = "Operation"
211

Robert Bradshaw's avatar
Robert Bradshaw committed
212 213 214 215 216 217
    def cpp_check(self, env):
        if not env.is_cpp():
            self.cpp_error()

    def cpp_error(self):
        error(self.pos, "%s only allowed in c++" % self.cpp_message)
218

219 220 221 222
    def clone_node(self):
        """Clone the node. This is defined as a shallow copy, except for member lists
           amongst the child attributes (from get_child_accessors) which are also
           copied. Lists containing child nodes are thus seen as a way for the node
Stefan Behnel's avatar
Stefan Behnel committed
223
           to hold multiple children directly; the list is not treated as a separate
224
           level in the tree."""
225 226 227
        result = copy.copy(self)
        for attrname in result.child_attrs:
            value = getattr(result, attrname)
228
            if isinstance(value, list):
229
                setattr(result, attrname, [x for x in value])
230
        return result
231 232


William Stein's avatar
William Stein committed
233
    #
234
    #  There are 3 phases of parse tree processing, applied in order to
William Stein's avatar
William Stein committed
235 236
    #  all the statements in a given scope-block:
    #
237
    #  (0) analyse_declarations
William Stein's avatar
William Stein committed
238 239 240 241
    #        Make symbol table entries for all declarations at the current
    #        level, both explicit (def, cdef, etc.) and implicit (assignment
    #        to an otherwise undeclared name).
    #
242
    #  (1) analyse_expressions
William Stein's avatar
William Stein committed
243 244
    #         Determine the result types of expressions and fill in the
    #         'type' attribute of each ExprNode. Insert coercion nodes into the
245
    #         tree where needed to convert to and from Python objects.
William Stein's avatar
William Stein committed
246 247 248 249
    #         Allocate temporary locals for intermediate results. Fill
    #         in the 'result_code' attribute of each ExprNode with a C code
    #         fragment.
    #
250
    #  (2) generate_code
William Stein's avatar
William Stein committed
251 252 253 254
    #         Emit C code for all declarations, statements and expressions.
    #         Recursively applies the 3 processing phases to the bodies of
    #         functions.
    #
255

William Stein's avatar
William Stein committed
256 257
    def analyse_declarations(self, env):
        pass
258

William Stein's avatar
William Stein committed
259 260 261
    def analyse_expressions(self, env):
        raise InternalError("analyse_expressions not implemented for %s" % \
            self.__class__.__name__)
262

William Stein's avatar
William Stein committed
263 264 265
    def generate_code(self, code):
        raise InternalError("generate_code not implemented for %s" % \
            self.__class__.__name__)
266

267 268 269 270
    def annotate(self, code):
        # mro does the wrong thing
        if isinstance(self, BlockNode):
            self.body.annotate(code)
271

272 273 274 275
    def end_pos(self):
        try:
            return self._end_pos
        except AttributeError:
Stefan Behnel's avatar
Stefan Behnel committed
276
            pos = self.pos
277 278 279
            if not self.child_attrs:
                self._end_pos = pos
                return pos
280
            for attr in self.child_attrs:
281
                child = getattr(self, attr)
282
                # Sometimes lists, sometimes nodes
283 284 285
                if child is None:
                    pass
                elif isinstance(child, list):
Stefan Behnel's avatar
Stefan Behnel committed
286 287
                    for c in child:
                        pos = max(pos, c.end_pos())
288
                else:
Stefan Behnel's avatar
Stefan Behnel committed
289 290 291
                    pos = max(pos, child.end_pos())
            self._end_pos = pos
            return pos
William Stein's avatar
William Stein committed
292

293
    def dump(self, level=0, filter_out=("pos",), cutoff=100, encountered=None):
294 295
        """Debug helper method that returns a recursive string representation of this node.
        """
296 297
        if cutoff == 0:
            return "<...nesting level cutoff...>"
298 299 300
        if encountered is None:
            encountered = set()
        if id(self) in encountered:
301
            return "<%s (0x%x) -- already output>" % (self.__class__.__name__, id(self))
302
        encountered.add(id(self))
303

304 305
        def dump_child(x, level):
            if isinstance(x, Node):
306
                return x.dump(level, filter_out, cutoff-1, encountered)
307
            elif isinstance(x, list):
Robert Bradshaw's avatar
Robert Bradshaw committed
308
                return "[%s]" % ", ".join([dump_child(item, level) for item in x])
309 310
            else:
                return repr(x)
311

312
        attrs = [(key, value) for key, value in self.__dict__.items() if key not in filter_out]
313
        if len(attrs) == 0:
314
            return "<%s (0x%x)>" % (self.__class__.__name__, id(self))
315 316
        else:
            indent = "  " * level
317
            res = "<%s (0x%x)\n" % (self.__class__.__name__, id(self))
318 319 320 321
            for key, value in attrs:
                res += "%s  %s: %s\n" % (indent, key, dump_child(value, level + 1))
            res += "%s>" % indent
            return res
322

323 324 325 326 327 328
    def dump_pos(self, mark_column=False, marker='(#)'):
        """Debug helper method that returns the source code context of this node as a string.
        """
        if not self.pos:
            return u''
        source_desc, line, col = self.pos
329
        contents = source_desc.get_lines(encoding='ASCII', error_handling='ignore')
330
        # line numbers start at 1
331
        lines = contents[max(0, line-3):line]
332 333 334 335 336 337 338 339
        current = lines[-1]
        if mark_column:
            current = current[:col] + marker + current[col:]
        lines[-1] = current.rstrip() + u'             # <<<<<<<<<<<<<<\n'
        lines += contents[line:line+2]
        return u'"%s":%d:%d\n%s\n' % (
            source_desc.get_escaped_description(), line, col, u''.join(lines))

340 341 342 343 344 345 346 347 348 349 350 351 352 353
class CompilerDirectivesNode(Node):
    """
    Sets compiler directives for the children nodes
    """
    #  directives     {string:value}  A dictionary holding the right value for
    #                                 *all* possible directives.
    #  body           Node
    child_attrs = ["body"]

    def analyse_declarations(self, env):
        old = env.directives
        env.directives = self.directives
        self.body.analyse_declarations(env)
        env.directives = old
354

355 356 357
    def analyse_expressions(self, env):
        old = env.directives
        env.directives = self.directives
358
        self.body = self.body.analyse_expressions(env)
359
        env.directives = old
360
        return self
361 362 363 364 365 366 367 368

    def generate_function_definitions(self, env, code):
        env_old = env.directives
        code_old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_function_definitions(env, code)
        env.directives = env_old
        code.globalstate.directives = code_old
369

370 371 372 373 374
    def generate_execution_code(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_execution_code(code)
        code.globalstate.directives = old
375

376 377 378 379 380
    def annotate(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.annotate(code)
        code.globalstate.directives = old
381

Stefan Behnel's avatar
Stefan Behnel committed
382
class BlockNode(object):
William Stein's avatar
William Stein committed
383 384
    #  Mixin class for nodes representing a declaration block.

385
    def generate_cached_builtins_decls(self, env, code):
386
        entries = env.global_scope().undeclared_cached_builtins
387
        for entry in entries:
388
            code.globalstate.add_cached_builtin_decl(entry)
389
        del entries[:]
390 391 392 393

    def generate_lambda_definitions(self, env, code):
        for node in env.lambda_defs:
            node.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
394 395 396

class StatListNode(Node):
    # stats     a list of StatNode
397

398
    child_attrs = ["stats"]
399

400
    @staticmethod
401 402
    def create_analysed(pos, env, *args, **kw):
        node = StatListNode(pos, *args, **kw)
403
        return node  # No node-specific analysis needed
404

William Stein's avatar
William Stein committed
405 406
    def analyse_declarations(self, env):
        #print "StatListNode.analyse_declarations" ###
407
        for stat in self.stats:
William Stein's avatar
William Stein committed
408
            stat.analyse_declarations(env)
409

William Stein's avatar
William Stein committed
410 411
    def analyse_expressions(self, env):
        #print "StatListNode.analyse_expressions" ###
412 413
        self.stats = [stat.analyse_expressions(env)
                      for stat in self.stats]
414
        return self
415

416
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
417 418
        #print "StatListNode.generate_function_definitions" ###
        for stat in self.stats:
419
            stat.generate_function_definitions(env, code)
420

William Stein's avatar
William Stein committed
421 422 423 424 425
    def generate_execution_code(self, code):
        #print "StatListNode.generate_execution_code" ###
        for stat in self.stats:
            code.mark_pos(stat.pos)
            stat.generate_execution_code(code)
426

427 428 429
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)
430

William Stein's avatar
William Stein committed
431 432 433 434 435 436 437 438 439 440 441 442 443

class StatNode(Node):
    #
    #  Code generation for statements is split into the following subphases:
    #
    #  (1) generate_function_definitions
    #        Emit C code for the definitions of any structs,
    #        unions, enums and functions defined in the current
    #        scope-block.
    #
    #  (2) generate_execution_code
    #        Emit C code for executable statements.
    #
444

445
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
446
        pass
447

William Stein's avatar
William Stein committed
448 449 450 451 452 453 454 455
    def generate_execution_code(self, code):
        raise InternalError("generate_execution_code not implemented for %s" % \
            self.__class__.__name__)


class CDefExternNode(StatNode):
    #  include_file   string or None
    #  body           StatNode
456

457
    child_attrs = ["body"]
458

William Stein's avatar
William Stein committed
459 460 461 462 463 464 465
    def analyse_declarations(self, env):
        if self.include_file:
            env.add_include_file(self.include_file)
        old_cinclude_flag = env.in_cinclude
        env.in_cinclude = 1
        self.body.analyse_declarations(env)
        env.in_cinclude = old_cinclude_flag
466

William Stein's avatar
William Stein committed
467
    def analyse_expressions(self, env):
468
        return self
469

William Stein's avatar
William Stein committed
470 471
    def generate_execution_code(self, code):
        pass
472 473 474

    def annotate(self, code):
        self.body.annotate(code)
475

William Stein's avatar
William Stein committed
476 477 478 479 480 481 482 483

class CDeclaratorNode(Node):
    # Part of a C declaration.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #      Returns (name, type) pair where name is the
484
    #      CNameDeclaratorNode of the name being declared
William Stein's avatar
William Stein committed
485 486
    #      and type is the type it is being declared as.
    #
487
    #  calling_convention  string   Calling convention of CFuncDeclaratorNode
488
    #                               for which this is a base
489

490 491
    child_attrs = []

492 493
    calling_convention = ""

494 495 496
    def analyse_templates(self):
        # Only C++ functions have templates.
        return None
William Stein's avatar
William Stein committed
497

498

William Stein's avatar
William Stein committed
499
class CNameDeclaratorNode(CDeclaratorNode):
500
    #  name    string             The Cython name being declared
Robert Bradshaw's avatar
Robert Bradshaw committed
501 502
    #  cname   string or None     C name, if specified
    #  default ExprNode or None   the value assigned on declaration
503

Robert Bradshaw's avatar
Robert Bradshaw committed
504
    child_attrs = ['default']
505

506
    default = None
507

508
    def analyse(self, base_type, env, nonempty=0):
509
        if nonempty and self.name == '':
510
            # May have mistaken the name for the type.
511
            if base_type.is_ptr or base_type.is_array or base_type.is_buffer:
512
                error(self.pos, "Missing argument name")
513 514
            elif base_type.is_void:
                error(self.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
515 516 517
            else:
                self.name = base_type.declaration_code("", for_display=1, pyrex=1)
                base_type = py_object_type
518 519 520 521

        if base_type.is_fused and env.fused_to_specific:
            base_type = base_type.specialize(env.fused_to_specific)

522
        self.type = base_type
William Stein's avatar
William Stein committed
523
        return self, base_type
524

525

William Stein's avatar
William Stein committed
526 527
class CPtrDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode
528

529 530
    child_attrs = ["base"]

531 532 533
    def analyse_templates(self):
        return self.base.analyse_templates()

534
    def analyse(self, base_type, env, nonempty=0):
William Stein's avatar
William Stein committed
535
        if base_type.is_pyobject:
536
            error(self.pos, "Pointer base type cannot be a Python object")
William Stein's avatar
William Stein committed
537
        ptr_type = PyrexTypes.c_ptr_type(base_type)
538 539
        return self.base.analyse(ptr_type, env, nonempty=nonempty)

Danilo Freitas's avatar
Danilo Freitas committed
540 541 542 543 544 545

class CReferenceDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

546 547 548
    def analyse_templates(self):
        return self.base.analyse_templates()

549
    def analyse(self, base_type, env, nonempty=0):
Danilo Freitas's avatar
Danilo Freitas committed
550
        if base_type.is_pyobject:
551
            error(self.pos, "Reference base type cannot be a Python object")
552
        ref_type = PyrexTypes.c_ref_type(base_type)
553 554
        return self.base.analyse(ref_type, env, nonempty=nonempty)

Danilo Freitas's avatar
Danilo Freitas committed
555

William Stein's avatar
William Stein committed
556 557 558
class CArrayDeclaratorNode(CDeclaratorNode):
    # base        CDeclaratorNode
    # dimension   ExprNode
559 560

    child_attrs = ["base", "dimension"]
561

562
    def analyse(self, base_type, env, nonempty=0):
563
        if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction:
564
            from .ExprNodes import TupleNode
Robert Bradshaw's avatar
Robert Bradshaw committed
565 566 567 568 569 570 571
            if isinstance(self.dimension, TupleNode):
                args = self.dimension.args
            else:
                args = self.dimension,
            values = [v.analyse_as_type(env) for v in args]
            if None in values:
                ix = values.index(None)
572 573 574 575
                error(args[ix].pos, "Template parameter not a type")
                base_type = error_type
            else:
                base_type = base_type.specialize_here(self.pos, values)
576
            return self.base.analyse(base_type, env, nonempty=nonempty)
William Stein's avatar
William Stein committed
577
        if self.dimension:
578
            self.dimension = self.dimension.analyse_const_expression(env)
William Stein's avatar
William Stein committed
579 580
            if not self.dimension.type.is_int:
                error(self.dimension.pos, "Array dimension not integer")
581 582 583 584 585 586 587
            size = self.dimension.get_constant_c_result_code()
            if size is not None:
                try:
                    size = int(size)
                except ValueError:
                    # runtime constant?
                    pass
William Stein's avatar
William Stein committed
588 589 590
        else:
            size = None
        if not base_type.is_complete():
591
            error(self.pos, "Array element type '%s' is incomplete" % base_type)
William Stein's avatar
William Stein committed
592
        if base_type.is_pyobject:
593
            error(self.pos, "Array element cannot be a Python object")
594
        if base_type.is_cfunction:
595
            error(self.pos, "Array element cannot be a function")
William Stein's avatar
William Stein committed
596
        array_type = PyrexTypes.c_array_type(base_type, size)
597
        return self.base.analyse(array_type, env, nonempty=nonempty)
William Stein's avatar
William Stein committed
598 599 600 601 602


class CFuncDeclaratorNode(CDeclaratorNode):
    # base             CDeclaratorNode
    # args             [CArgDeclNode]
603
    # templates        [TemplatePlaceholderType]
William Stein's avatar
William Stein committed
604 605 606
    # has_varargs      boolean
    # exception_value  ConstNode
    # exception_check  boolean    True if PyErr_Occurred check needed
607 608
    # nogil            boolean    Can be called without gil
    # with_gil         boolean    Acquire gil around function body
609
    # is_const_method  boolean    Whether this is a const method
610

611 612
    child_attrs = ["base", "args", "exception_value"]

613
    overridable = 0
614
    optional_arg_count = 0
615
    is_const_method = 0
616 617 618 619
    templates = None

    def analyse_templates(self):
        if isinstance(self.base, CArrayDeclaratorNode):
620
            from .ExprNodes import TupleNode, NameNode
621 622 623 624 625 626 627
            template_node = self.base.dimension
            if isinstance(template_node, TupleNode):
                template_nodes = template_node.args
            elif isinstance(template_node, NameNode):
                template_nodes = [template_node]
            else:
                error(template_node.pos, "Template arguments must be a list of names")
628
                return None
629 630 631 632 633 634 635 636 637 638
            self.templates = []
            for template in template_nodes:
                if isinstance(template, NameNode):
                    self.templates.append(PyrexTypes.TemplatePlaceholderType(template.name))
                else:
                    error(template.pos, "Template arguments must be a list of names")
            self.base = self.base.base
            return self.templates
        else:
            return None
William Stein's avatar
William Stein committed
639

640 641 642
    def analyse(self, return_type, env, nonempty=0, directive_locals=None):
        if directive_locals is None:
            directive_locals = {}
643 644
        if nonempty:
            nonempty -= 1
William Stein's avatar
William Stein committed
645
        func_type_args = []
646
        for i, arg_node in enumerate(self.args):
647
            name_declarator, type = arg_node.analyse(
648 649
                env, nonempty=nonempty,
                is_self_arg=(i == 0 and env.is_c_class_scope and 'staticmethod' not in env.directives))
William Stein's avatar
William Stein committed
650
            name = name_declarator.name
651 652 653 654 655
            if name in directive_locals:
                type_node = directive_locals[name]
                other_type = type_node.analyse_as_type(env)
                if other_type is None:
                    error(type_node.pos, "Not a type")
Robert Bradshaw's avatar
Robert Bradshaw committed
656
                elif (type is not PyrexTypes.py_object_type
657 658 659 660 661
                      and not type.same_as(other_type)):
                    error(self.base.pos, "Signature does not agree with previous declaration")
                    error(type_node.pos, "Previous declaration here")
                else:
                    type = other_type
William Stein's avatar
William Stein committed
662
            if name_declarator.cname:
663 664
                error(self.pos, "Function argument cannot have C name specification")
            if i == 0 and env.is_c_class_scope and type.is_unspecified:
665 666
                # fix the type of self
                type = env.parent_type
William Stein's avatar
William Stein committed
667 668 669 670 671
            # Turn *[] argument into **
            if type.is_array:
                type = PyrexTypes.c_ptr_type(type.base_type)
            # Catch attempted C-style func(void) decl
            if type.is_void:
Robert Bradshaw's avatar
Robert Bradshaw committed
672
                error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
William Stein's avatar
William Stein committed
673 674 675
            func_type_args.append(
                PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
            if arg_node.default:
676
                self.optional_arg_count += 1
677 678
            elif self.optional_arg_count:
                error(self.pos, "Non-default argument follows default argument")
679

William Stein's avatar
William Stein committed
680 681
        exc_val = None
        exc_check = 0
682
        if self.exception_check == '+':
683
            env.add_include_file('ios')         # for std::ios_base::failure
684
            env.add_include_file('new')         # for std::bad_alloc
685
            env.add_include_file('stdexcept')
686
            env.add_include_file('typeinfo')    # for std::bad_cast
687 688 689
        if (return_type.is_pyobject
                and (self.exception_value or self.exception_check)
                and self.exception_check != '+'):
690
            error(self.pos, "Exception clause not allowed for function returning Python object")
William Stein's avatar
William Stein committed
691 692
        else:
            if self.exception_value:
693
                self.exception_value = self.exception_value.analyse_const_expression(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
694 695
                if self.exception_check == '+':
                    exc_val_type = self.exception_value.type
696 697 698 699 700
                    if (not exc_val_type.is_error
                            and not exc_val_type.is_pyobject
                            and not (exc_val_type.is_cfunction
                                     and not exc_val_type.return_type.is_pyobject
                                     and not exc_val_type.args)):
Robert Bradshaw's avatar
Robert Bradshaw committed
701
                        error(self.exception_value.pos,
702
                              "Exception value must be a Python exception or cdef function with no arguments.")
703
                    exc_val = self.exception_value
Robert Bradshaw's avatar
Robert Bradshaw committed
704
                else:
705 706 707 708 709 710 711 712 713 714
                    self.exception_value = self.exception_value.coerce_to(
                        return_type, env).analyse_const_expression(env)
                    exc_val = self.exception_value.get_constant_c_result_code()
                    if exc_val is None:
                        raise InternalError(
                            "get_constant_c_result_code not implemented for %s" %
                            self.exception_value.__class__.__name__)
                    if not return_type.assignable_from(self.exception_value.type):
                        error(self.exception_value.pos,
                              "Exception value incompatible with function return type")
William Stein's avatar
William Stein committed
715
            exc_check = self.exception_check
716
        if return_type.is_cfunction:
717
            error(self.pos, "Function cannot return a function")
William Stein's avatar
William Stein committed
718
        func_type = PyrexTypes.CFuncType(
719
            return_type, func_type_args, self.has_varargs,
720 721 722 723 724 725
            optional_arg_count=self.optional_arg_count,
            exception_value=exc_val, exception_check=exc_check,
            calling_convention=self.base.calling_convention,
            nogil=self.nogil, with_gil=self.with_gil, is_overridable=self.overridable,
            is_const_method=self.is_const_method,
            templates=self.templates)
726

727
        if self.optional_arg_count:
728 729 730 731 732 733 734 735 736 737 738
            if func_type.is_fused:
                # This is a bit of a hack... When we need to create specialized CFuncTypes
                # on the fly because the cdef is defined in a pxd, we need to declare the specialized optional arg
                # struct
                def declare_opt_arg_struct(func_type, fused_cname):
                    self.declare_optional_arg_struct(func_type, env, fused_cname)

                func_type.declare_opt_arg_struct = declare_opt_arg_struct
            else:
                self.declare_optional_arg_struct(func_type, env)

739 740 741 742 743 744 745
        callspec = env.directives['callspec']
        if callspec:
            current = func_type.calling_convention
            if current and current != callspec:
                error(self.pos, "cannot have both '%s' and '%s' "
                      "calling conventions" % (current, callspec))
            func_type.calling_convention = callspec
William Stein's avatar
William Stein committed
746 747
        return self.base.analyse(func_type, env)

748 749 750 751 752 753 754 755 756 757 758
    def declare_optional_arg_struct(self, func_type, env, fused_cname=None):
        """
        Declares the optional argument struct (the struct used to hold the
        values for optional arguments). For fused cdef functions, this is
        deferred as analyse_declarations is called only once (on the fused
        cdef function).
        """
        scope = StructOrUnionScope()
        arg_count_member = '%sn' % Naming.pyrex_prefix
        scope.declare_var(arg_count_member, PyrexTypes.c_int_type, self.pos)

759 760
        for arg in func_type.args[len(func_type.args) - self.optional_arg_count:]:
            scope.declare_var(arg.name, arg.type, arg.pos, allow_pyobject=1)
761 762 763 764 765 766 767

        struct_cname = env.mangle(Naming.opt_arg_prefix, self.base.name)

        if fused_cname is not None:
            struct_cname = PyrexTypes.get_fused_cname(fused_cname, struct_cname)

        op_args_struct = env.global_scope().declare_struct_or_union(
768 769 770 771 772 773
            name=struct_cname,
            kind='struct',
            scope=scope,
            typedef_flag=0,
            pos=self.pos,
            cname=struct_cname)
774 775 776 777 778 779

        op_args_struct.defined_in_pxd = 1
        op_args_struct.used = 1

        func_type.op_arg_struct = PyrexTypes.c_ptr_type(op_args_struct.type)

William Stein's avatar
William Stein committed
780

781 782 783 784 785
class CConstDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

786
    def analyse(self, base_type, env, nonempty=0):
787 788 789 790
        if base_type.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        const = PyrexTypes.c_const_type(base_type)
791
        return self.base.analyse(const, env, nonempty=nonempty)
792 793


William Stein's avatar
William Stein committed
794 795 796 797 798 799
class CArgDeclNode(Node):
    # Item in a function declaration argument list.
    #
    # base_type      CBaseTypeNode
    # declarator     CDeclaratorNode
    # not_none       boolean            Tagged with 'not None'
800 801
    # or_none        boolean            Tagged with 'or None'
    # accept_none    boolean            Resolved boolean for not_none/or_none
William Stein's avatar
William Stein committed
802
    # default        ExprNode or None
803
    # default_value  PyObjectConst      constant for default value
804
    # annotation     ExprNode or None   Py3 function arg annotation
William Stein's avatar
William Stein committed
805
    # is_self_arg    boolean            Is the "self" arg of an extension type method
806
    # is_type_arg    boolean            Is the "class" arg of an extension type classmethod
807
    # is_kw_only     boolean            Is a keyword-only argument
808
    # is_dynamic     boolean            Non-literal arg stored inside CyFunction
809

810
    child_attrs = ["base_type", "declarator", "default", "annotation"]
811

William Stein's avatar
William Stein committed
812
    is_self_arg = 0
813
    is_type_arg = 0
814
    is_generic = 1
815 816 817
    kw_only = 0
    not_none = 0
    or_none = 0
818 819
    type = None
    name_declarator = None
820
    default_value = None
821
    annotation = None
822
    is_dynamic = 0
823

824
    def analyse(self, env, nonempty=0, is_self_arg=False):
825 826
        if is_self_arg:
            self.base_type.is_self_arg = self.is_self_arg = True
827
        if self.type is None:
Stefan Behnel's avatar
Stefan Behnel committed
828
            # The parser may misinterpret names as types. We fix that here.
829 830
            if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '':
                if nonempty:
831 832
                    if self.base_type.is_basic_c_type:
                        # char, short, long called "int"
833
                        type = self.base_type.analyse(env, could_be_name=True)
834
                        arg_name = type.empty_declaration_code()
835 836 837
                    else:
                        arg_name = self.base_type.name
                    self.declarator.name = EncodedString(arg_name)
838 839 840 841 842
                    self.base_type.name = None
                    self.base_type.is_basic_c_type = False
                could_be_name = True
            else:
                could_be_name = False
843
            self.base_type.is_arg = True
844
            base_type = self.base_type.analyse(env, could_be_name=could_be_name)
Robert Bradshaw's avatar
Robert Bradshaw committed
845
            if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name:
846
                self.declarator.name = self.base_type.arg_name
847

848 849
            # The parser is unable to resolve the ambiguity of [] as part of the
            # type (e.g. in buffers) or empty declarator (as with arrays).
850
            # This is only arises for empty multi-dimensional arrays.
851 852
            if (base_type.is_array
                    and isinstance(self.base_type, TemplatedTypeNode)
853 854 855 856 857 858
                    and isinstance(self.declarator, CArrayDeclaratorNode)):
                declarator = self.declarator
                while isinstance(declarator.base, CArrayDeclaratorNode):
                    declarator = declarator.base
                declarator.base = self.base_type.array_declarator
                base_type = base_type.base_type
859 860 861 862 863 864 865

            # inject type declaration from annotations
            if self.annotation and env.directives['annotation_typing'] and self.base_type.name is None:
                arg_type = self.inject_type_from_annotations(env)
                if arg_type is not None:
                    base_type = arg_type
            return self.declarator.analyse(base_type, env, nonempty=nonempty)
866
        else:
867
            return self.name_declarator, self.type
William Stein's avatar
William Stein committed
868

869 870 871
    def inject_type_from_annotations(self, env):
        annotation = self.annotation
        if not annotation:
872
            return None
873 874 875
        base_type, arg_type = _analyse_signature_annotation(annotation, env)
        if base_type is not None:
            self.base_type = base_type
876 877
        return arg_type

878 879 880 881 882 883 884 885 886
    def calculate_default_value_code(self, code):
        if self.default_value is None:
            if self.default:
                if self.default.is_literal:
                    # will not output any code, just assign the result_code
                    self.default.generate_evaluation_code(code)
                    return self.type.cast_code(self.default.result())
                self.default_value = code.get_argument_default_const(self.type)
        return self.default_value
887

888 889 890 891
    def annotate(self, code):
        if self.default:
            self.default.annotate(code)

892
    def generate_assignment_code(self, code, target=None, overloaded_assignment=False):
893 894 895 896 897 898 899
        default = self.default
        if default is None or default.is_literal:
            return
        if target is None:
            target = self.calculate_default_value_code(code)
        default.generate_evaluation_code(code)
        default.make_owned_reference(code)
900
        result = default.result() if overloaded_assignment else default.result_as(self.type)
901 902 903 904 905 906
        code.putln("%s = %s;" % (target, result))
        if self.type.is_pyobject:
            code.put_giveref(default.result())
        default.generate_post_assignment_code(code)
        default.free_temps(code)

William Stein's avatar
William Stein committed
907 908 909 910 911 912 913 914

class CBaseTypeNode(Node):
    # Abstract base class for C base type nodes.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #     Returns the type.
915

916 917
    def analyse_as_type(self, env):
        return self.analyse(env)
918

Stefan Behnel's avatar
Stefan Behnel committed
919

920 921
class CAnalysedBaseTypeNode(Node):
    # type            type
922

923
    child_attrs = []
924

925
    def analyse(self, env, could_be_name=False):
926
        return self.type
William Stein's avatar
William Stein committed
927

Stefan Behnel's avatar
Stefan Behnel committed
928

William Stein's avatar
William Stein committed
929 930 931 932 933 934
class CSimpleBaseTypeNode(CBaseTypeNode):
    # name             string
    # module_path      [string]     Qualifying name components
    # is_basic_c_type  boolean
    # signed           boolean
    # longness         integer
935
    # complex          boolean
William Stein's avatar
William Stein committed
936
    # is_self_arg      boolean      Is self argument of C method
937
    # ##is_type_arg      boolean      Is type argument of class method
William Stein's avatar
William Stein committed
938

939
    child_attrs = []
940
    arg_name = None   # in case the argument name was interpreted as a type
941 942 943
    module_path = []
    is_basic_c_type = False
    complex = False
944

945
    def analyse(self, env, could_be_name=False):
William Stein's avatar
William Stein committed
946
        # Return type descriptor.
947
        #print "CSimpleBaseTypeNode.analyse: is_self_arg =", self.is_self_arg ###
William Stein's avatar
William Stein committed
948 949 950 951 952 953 954 955 956
        type = None
        if self.is_basic_c_type:
            type = PyrexTypes.simple_c_type(self.signed, self.longness, self.name)
            if not type:
                error(self.pos, "Unrecognised type modifier combination")
        elif self.name == "object" and not self.module_path:
            type = py_object_type
        elif self.name is None:
            if self.is_self_arg and env.is_c_class_scope:
957
                #print "CSimpleBaseTypeNode.analyse: defaulting to parent type" ###
William Stein's avatar
William Stein committed
958
                type = env.parent_type
959 960
            ## elif self.is_type_arg and env.is_c_class_scope:
            ##     type = Builtin.type_type
William Stein's avatar
William Stein committed
961 962 963
            else:
                type = py_object_type
        else:
964
            if self.module_path:
965 966 967 968
                # Maybe it's a nested C++ class.
                scope = env
                for item in self.module_path:
                    entry = scope.lookup(item)
Robert Bradshaw's avatar
Robert Bradshaw committed
969
                    if entry is not None and entry.is_cpp_class:
970 971 972 973
                        scope = entry.type.scope
                    else:
                        scope = None
                        break
974

975 976 977 978 979
                if scope is None:
                    # Maybe it's a cimport.
                    scope = env.find_imported_module(self.module_path, self.pos)
                    if scope:
                        scope.fused_to_specific = env.fused_to_specific
980 981
            else:
                scope = env
982

William Stein's avatar
William Stein committed
983
            if scope:
984 985
                if scope.is_c_class_scope:
                    scope = scope.global_scope()
986 987 988 989

                type = scope.lookup_type(self.name)
                if type is not None:
                    pass
990 991 992
                elif could_be_name:
                    if self.is_self_arg and env.is_c_class_scope:
                        type = env.parent_type
993 994
                    ## elif self.is_type_arg and env.is_c_class_scope:
                    ##     type = Builtin.type_type
995 996
                    else:
                        type = py_object_type
997
                    self.arg_name = EncodedString(self.name)
William Stein's avatar
William Stein committed
998
                else:
Danilo Freitas's avatar
Danilo Freitas committed
999 1000 1001
                    if self.templates:
                        if not self.name in self.templates:
                            error(self.pos, "'%s' is not a type identifier" % self.name)
1002
                        type = PyrexTypes.TemplatePlaceholderType(self.name)
Danilo Freitas's avatar
Danilo Freitas committed
1003 1004
                    else:
                        error(self.pos, "'%s' is not a type identifier" % self.name)
1005
        if self.complex:
1006 1007 1008
            if not type.is_numeric or type.is_complex:
                error(self.pos, "can only complexify c numeric types")
            type = PyrexTypes.CComplexType(type)
1009
            type.create_declaration_utility_code(env)
1010 1011 1012 1013 1014 1015 1016 1017
        elif type is Builtin.complex_type:
            # Special case: optimise builtin complex type into C's
            # double complex.  The parser cannot do this (as for the
            # normal scalar types) as the user may have redeclared the
            # 'complex' type.  Testing for the exact type here works.
            type = PyrexTypes.c_double_complex_type
            type.create_declaration_utility_code(env)
            self.complex = True
William Stein's avatar
William Stein committed
1018 1019 1020 1021 1022
        if type:
            return type
        else:
            return PyrexTypes.error_type

1023
class MemoryViewSliceTypeNode(CBaseTypeNode):
1024

1025
    name = 'memoryview'
1026 1027
    child_attrs = ['base_type_node', 'axes']

1028
    def analyse(self, env, could_be_name=False):
1029 1030 1031 1032

        base_type = self.base_type_node.analyse(env)
        if base_type.is_error: return base_type

1033
        from . import MemoryView
1034 1035 1036

        try:
            axes_specs = MemoryView.get_axes_specs(env, self.axes)
1037
        except CompileError as e:
1038 1039 1040 1041
            error(e.position, e.message_only)
            self.type = PyrexTypes.ErrorType()
            return self.type

1042 1043 1044 1045
        if not MemoryView.validate_axes(self.pos, axes_specs):
            self.type = error_type
        else:
            self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
1046
            self.type.validate_memslice_dtype(self.pos)
1047
            self.use_memview_utilities(env)
1048

1049
        return self.type
1050

1051
    def use_memview_utilities(self, env):
1052
        from . import MemoryView
1053 1054 1055
        env.use_utility_code(MemoryView.view_utility_code)


Robert Bradshaw's avatar
Robert Bradshaw committed
1056
class CNestedBaseTypeNode(CBaseTypeNode):
1057
    # For C++ classes that live inside other C++ classes.
Robert Bradshaw's avatar
Robert Bradshaw committed
1058 1059 1060

    # name             string
    # base_type        CBaseTypeNode
Stefan Behnel's avatar
Stefan Behnel committed
1061

Robert Bradshaw's avatar
Robert Bradshaw committed
1062
    child_attrs = ['base_type']
Stefan Behnel's avatar
Stefan Behnel committed
1063

1064
    def analyse(self, env, could_be_name=None):
Robert Bradshaw's avatar
Robert Bradshaw committed
1065 1066 1067 1068 1069 1070 1071 1072 1073 1074 1075 1076
        base_type = self.base_type.analyse(env)
        if base_type is PyrexTypes.error_type:
            return PyrexTypes.error_type
        if not base_type.is_cpp_class:
            error(self.pos, "'%s' is not a valid type scope" % base_type)
            return PyrexTypes.error_type
        type_entry = base_type.scope.lookup_here(self.name)
        if not type_entry or not type_entry.is_type:
            error(self.pos, "'%s.%s' is not a type identifier" % (base_type, self.name))
            return PyrexTypes.error_type
        return type_entry.type

1077

Danilo Freitas's avatar
Danilo Freitas committed
1078
class TemplatedTypeNode(CBaseTypeNode):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1079
    #  After parsing:
1080 1081
    #  positional_args  [ExprNode]        List of positional arguments
    #  keyword_args     DictNode          Keyword arguments
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1082 1083 1084
    #  base_type_node   CBaseTypeNode

    #  After analysis:
1085
    #  type             PyrexTypes.BufferType or PyrexTypes.CppClassType  ...containing the right options
1086 1087 1088

    child_attrs = ["base_type_node", "positional_args",
                   "keyword_args", "dtype_node"]
1089

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1090
    dtype_node = None
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1091 1092

    name = None
1093

1094
    def analyse(self, env, could_be_name=False, base_type=None):
1095 1096
        if base_type is None:
            base_type = self.base_type_node.analyse(env)
1097
        if base_type.is_error: return base_type
1098

1099
        if base_type.is_cpp_class and base_type.is_template_type():
1100
            # Templated class
1101
            if self.keyword_args and self.keyword_args.key_value_pairs:
Stefan Behnel's avatar
Stefan Behnel committed
1102
                error(self.pos, "c++ templates cannot take keyword arguments")
1103 1104 1105 1106
                self.type = PyrexTypes.error_type
            else:
                template_types = []
                for template_node in self.positional_args:
1107 1108 1109 1110 1111
                    type = template_node.analyse_as_type(env)
                    if type is None:
                        error(template_node.pos, "unknown type in template argument")
                        return error_type
                    template_types.append(type)
1112
                self.type = base_type.specialize_here(self.pos, template_types)
1113

1114 1115
        elif base_type.is_pyobject:
            # Buffer
1116
            from . import Buffer
1117

1118 1119 1120 1121 1122 1123
            options = Buffer.analyse_buffer_options(
                self.pos,
                env,
                self.positional_args,
                self.keyword_args,
                base_type.buffer_defaults)
1124

Robert Bradshaw's avatar
Robert Bradshaw committed
1125 1126
            if sys.version_info[0] < 3:
                # Py 2.x enforces byte strings as keyword arguments ...
1127 1128
                options = dict([(name.encode('ASCII'), value)
                                for name, value in options.items()])
Stefan Behnel's avatar
Stefan Behnel committed
1129

1130
            self.type = PyrexTypes.BufferType(base_type, **options)
1131 1132
            if has_np_pythran(env) and is_pythran_buffer(self.type):
                self.type = PyrexTypes.PythranExpr(pythran_type(self.type), self.type)
1133

1134 1135
        else:
            # Array
1136
            empty_declarator = CNameDeclaratorNode(self.pos, name="", cname=None)
1137 1138 1139 1140
            if len(self.positional_args) > 1 or self.keyword_args.key_value_pairs:
                error(self.pos, "invalid array declaration")
                self.type = PyrexTypes.error_type
            else:
1141
                # It would be nice to merge this class with CArrayDeclaratorNode,
1142 1143 1144 1145 1146
                # but arrays are part of the declaration, not the type...
                if not self.positional_args:
                    dimension = None
                else:
                    dimension = self.positional_args[0]
1147 1148 1149 1150
                self.array_declarator = CArrayDeclaratorNode(
                    self.pos,
                    base=empty_declarator,
                    dimension=dimension)
1151
                self.type = self.array_declarator.analyse(base_type, env)[1]
1152

1153 1154 1155
        if self.type.is_fused and env.fused_to_specific:
            self.type = self.type.specialize(env.fused_to_specific)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1156
        return self.type
William Stein's avatar
William Stein committed
1157

1158

William Stein's avatar
William Stein committed
1159 1160 1161
class CComplexBaseTypeNode(CBaseTypeNode):
    # base_type   CBaseTypeNode
    # declarator  CDeclaratorNode
1162

1163 1164
    child_attrs = ["base_type", "declarator"]

1165
    def analyse(self, env, could_be_name=False):
1166
        base = self.base_type.analyse(env, could_be_name)
William Stein's avatar
William Stein committed
1167 1168 1169 1170
        _, type = self.declarator.analyse(base, env)
        return type


1171 1172 1173 1174 1175
class CTupleBaseTypeNode(CBaseTypeNode):
    # components [CBaseTypeNode]

    child_attrs = ["components"]

Robert Bradshaw's avatar
Robert Bradshaw committed
1176
    def analyse(self, env, could_be_name=False):
1177 1178 1179 1180
        component_types = []
        for c in self.components:
            type = c.analyse(env)
            if type.is_pyobject:
1181 1182
                error(c.pos, "Tuple types can't (yet) contain Python objects.")
                return error_type
1183
            component_types.append(type)
1184
        entry = env.declare_tuple_type(self.pos, component_types)
Robert Bradshaw's avatar
Robert Bradshaw committed
1185
        entry.used = True
1186
        return entry.type
1187 1188


1189 1190 1191 1192 1193 1194
class FusedTypeNode(CBaseTypeNode):
    """
    Represents a fused type in a ctypedef statement:

        ctypedef cython.fused_type(int, long, long long) integral

Mark Florisson's avatar
Mark Florisson committed
1195
    name            str                     name of this fused type
1196 1197 1198 1199 1200
    types           [CSimpleBaseTypeNode]   is the list of types to be fused
    """

    child_attrs = []

Mark Florisson's avatar
Mark Florisson committed
1201 1202 1203 1204 1205 1206 1207
    def analyse_declarations(self, env):
        type = self.analyse(env)
        entry = env.declare_typedef(self.name, type, self.pos)

        # Omit the typedef declaration that self.declarator would produce
        entry.in_cinclude = True

1208
    def analyse(self, env, could_be_name=False):
Mark Florisson's avatar
Mark Florisson committed
1209 1210 1211
        types = []
        for type_node in self.types:
            type = type_node.analyse_as_type(env)
1212

Mark Florisson's avatar
Mark Florisson committed
1213 1214 1215
            if not type:
                error(type_node.pos, "Not a type")
                continue
1216

Mark Florisson's avatar
Mark Florisson committed
1217
            if type in types:
1218 1219
                error(type_node.pos, "Type specified multiple times")
            else:
Mark Florisson's avatar
Mark Florisson committed
1220
                types.append(type)
1221

1222 1223
        # if len(self.types) == 1:
        #     return types[0]
Mark Florisson's avatar
Mark Florisson committed
1224 1225

        return PyrexTypes.FusedType(types, name=self.name)
1226

1227

Robert Bradshaw's avatar
Robert Bradshaw committed
1228 1229 1230 1231 1232
class CConstTypeNode(CBaseTypeNode):
    # base_type     CBaseTypeNode

    child_attrs = ["base_type"]

1233
    def analyse(self, env, could_be_name=False):
Robert Bradshaw's avatar
Robert Bradshaw committed
1234 1235 1236 1237 1238 1239 1240
        base = self.base_type.analyse(env, could_be_name)
        if base.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        return PyrexTypes.c_const_type(base)


William Stein's avatar
William Stein committed
1241 1242 1243 1244 1245 1246
class CVarDefNode(StatNode):
    #  C variable definition or forward/extern function declaration.
    #
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarators   [CDeclaratorNode]
1247
    #  in_pxd        boolean
Stefan Behnel's avatar
Stefan Behnel committed
1248
    #  api           boolean
1249
    #  overridable   boolean        whether it is a cpdef
1250
    #  modifiers     ['inline']
1251

1252
    #  decorators    [cython.locals(...)] or None
1253
    #  directive_locals { string : NameNode } locals defined by cython.locals(...)
1254 1255

    child_attrs = ["base_type", "declarators"]
1256

Robert Bradshaw's avatar
Robert Bradshaw committed
1257
    decorators = None
1258
    directive_locals = None
1259

1260
    def analyse_declarations(self, env, dest_scope=None):
1261 1262
        if self.directive_locals is None:
            self.directive_locals = {}
William Stein's avatar
William Stein committed
1263 1264
        if not dest_scope:
            dest_scope = env
1265
        self.dest_scope = dest_scope
1266 1267 1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280

        if self.declarators:
            templates = self.declarators[0].analyse_templates()
        else:
            templates = None
        if templates is not None:
            if self.visibility != 'extern':
                error(self.pos, "Only extern functions allowed")
            if len(self.declarators) > 1:
                error(self.declarators[1].pos, "Can't multiply declare template types")
            env = TemplateScope('func_template', env)
            env.directives = env.outer_scope.directives
            for template_param in templates:
                env.declare_type(template_param.name, template_param, self.pos)

William Stein's avatar
William Stein committed
1281
        base_type = self.base_type.analyse(env)
1282

1283 1284
        if base_type.is_fused and not self.in_pxd and (env.is_c_class_scope or
                                                       env.is_module_scope):
1285 1286 1287
            error(self.pos, "Fused types not allowed here")
            return error_type

1288
        self.entry = None
1289
        visibility = self.visibility
1290

William Stein's avatar
William Stein committed
1291
        for declarator in self.declarators:
1292

1293
            if (len(self.declarators) > 1
1294 1295 1296 1297 1298
                    and not isinstance(declarator, CNameDeclaratorNode)
                    and env.directives['warn.multiple_declarators']):
                warning(
                    declarator.pos,
                    "Non-trivial type declarators in shared declaration (e.g. mix of pointers and values). "
1299
                    "Each pointer declaration should be on its own line.", 1)
1300

1301
            create_extern_wrapper = (self.overridable
1302 1303
                                     and self.visibility == 'extern'
                                     and env.is_module_scope)
1304 1305
            if create_extern_wrapper:
                declarator.overridable = False
1306 1307 1308 1309
            if isinstance(declarator, CFuncDeclaratorNode):
                name_declarator, type = declarator.analyse(base_type, env, directive_locals=self.directive_locals)
            else:
                name_declarator, type = declarator.analyse(base_type, env)
William Stein's avatar
William Stein committed
1310
            if not type.is_complete():
1311
                if not (self.visibility == 'extern' and type.is_array or type.is_memoryviewslice):
1312
                    error(declarator.pos, "Variable type '%s' is incomplete" % type)
William Stein's avatar
William Stein committed
1313
            if self.visibility == 'extern' and type.is_pyobject:
1314
                error(declarator.pos, "Python object cannot be declared extern")
William Stein's avatar
William Stein committed
1315 1316
            name = name_declarator.name
            cname = name_declarator.cname
1317 1318 1319
            if name == '':
                error(declarator.pos, "Missing name in declaration.")
                return
1320 1321
            if type.is_reference and self.visibility != 'extern':
                error(declarator.pos, "C++ references cannot be declared; use a pointer instead")
William Stein's avatar
William Stein committed
1322
            if type.is_cfunction:
1323 1324
                if 'staticmethod' in env.directives:
                    type.is_static_method = True
1325 1326
                self.entry = dest_scope.declare_cfunction(
                    name, type, declarator.pos,
1327 1328
                    cname=cname, visibility=self.visibility, in_pxd=self.in_pxd,
                    api=self.api, modifiers=self.modifiers, overridable=self.overridable)
1329 1330
                if self.entry is not None:
                    self.entry.directive_locals = copy.copy(self.directive_locals)
1331
                if create_extern_wrapper:
1332
                    self.entry.type.create_to_py_utility_code(env)
1333
                    self.entry.create_wrapper = True
William Stein's avatar
William Stein committed
1334
            else:
1335
                if self.directive_locals:
1336
                    error(self.pos, "Decorators can only be followed by functions")
1337 1338 1339 1340
                self.entry = dest_scope.declare_var(
                    name, type, declarator.pos,
                    cname=cname, visibility=visibility, in_pxd=self.in_pxd,
                    api=self.api, is_cdef=1)
1341 1342
                if Options.docstrings:
                    self.entry.doc = embed_position(self.pos, self.doc)
1343

William Stein's avatar
William Stein committed
1344 1345 1346 1347 1348 1349

class CStructOrUnionDefNode(StatNode):
    #  name          string
    #  cname         string or None
    #  kind          "struct" or "union"
    #  typedef_flag  boolean
1350
    #  visibility    "public" or "private"
1351
    #  api           boolean
Stefan Behnel's avatar
Stefan Behnel committed
1352
    #  in_pxd        boolean
William Stein's avatar
William Stein committed
1353 1354
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1355
    #  packed        boolean
1356

1357
    child_attrs = ["attributes"]
Vitja Makarov's avatar
Vitja Makarov committed
1358

1359
    def declare(self, env, scope=None):
William Stein's avatar
William Stein committed
1360 1361
        self.entry = env.declare_struct_or_union(
            self.name, self.kind, scope, self.typedef_flag, self.pos,
1362 1363
            self.cname, visibility=self.visibility, api=self.api,
            packed=self.packed)
1364 1365 1366 1367 1368 1369

    def analyse_declarations(self, env):
        scope = None
        if self.attributes is not None:
            scope = StructOrUnionScope(self.name)
        self.declare(env, scope)
William Stein's avatar
William Stein committed
1370
        if self.attributes is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1371 1372
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
William Stein's avatar
William Stein committed
1373 1374
            for attr in self.attributes:
                attr.analyse_declarations(env, scope)
1375 1376 1377 1378
            if self.visibility != 'extern':
                for attr in scope.var_entries:
                    type = attr.type
                    while type.is_array:
1379 1380
                        type = type.base_type
                    if type == self.entry.type:
1381
                        error(attr.pos, "Struct cannot contain itself as a member.")
1382

William Stein's avatar
William Stein committed
1383
    def analyse_expressions(self, env):
1384
        return self
1385

William Stein's avatar
William Stein committed
1386 1387 1388 1389
    def generate_execution_code(self, code):
        pass


1390
class CppClassNode(CStructOrUnionDefNode, BlockNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
1391 1392 1393

    #  name          string
    #  cname         string or None
1394
    #  visibility    "extern"
Robert Bradshaw's avatar
Robert Bradshaw committed
1395 1396 1397
    #  in_pxd        boolean
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1398
    #  base_classes  [CBaseTypeNode]
1399
    #  templates     [(string, bool)] or None
1400 1401 1402
    #  decorators    [DecoratorNode] or None

    decorators = None
Robert Bradshaw's avatar
Robert Bradshaw committed
1403

1404 1405 1406 1407
    def declare(self, env):
        if self.templates is None:
            template_types = None
        else:
1408 1409 1410 1411 1412
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
                              for template_name, required in self.templates]
            num_optional_templates = sum(not required for _, required in self.templates)
            if num_optional_templates and not all(required for _, required in self.templates[:-num_optional_templates]):
                error(self.pos, "Required template parameters must precede optional template parameters.")
1413
        self.entry = env.declare_cpp_class(
1414 1415
            self.name, None, self.pos, self.cname,
            base_classes=[], visibility=self.visibility, templates=template_types)
1416

Robert Bradshaw's avatar
Robert Bradshaw committed
1417
    def analyse_declarations(self, env):
1418 1419 1420 1421 1422 1423
        if self.templates is None:
            template_types = template_names = None
        else:
            template_names = [template_name for template_name, _ in self.templates]
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name, not required)
                              for template_name, required in self.templates]
Robert Bradshaw's avatar
Robert Bradshaw committed
1424
        scope = None
1425
        if self.attributes is not None:
1426
            scope = CppClassScope(self.name, env, templates=template_names)
1427 1428 1429 1430 1431 1432
        def base_ok(base_class):
            if base_class.is_cpp_class or base_class.is_struct:
                return True
            else:
                error(self.pos, "Base class '%s' not a struct or class." % base_class)
        base_class_types = filter(base_ok, [b.analyse(scope or env) for b in self.base_classes])
1433
        self.entry = env.declare_cpp_class(
1434
            self.name, scope, self.pos,
1435
            self.cname, base_class_types, visibility=self.visibility, templates=template_types)
1436 1437
        if self.entry is None:
            return
Danilo Freitas's avatar
Danilo Freitas committed
1438
        self.entry.is_cpp_class = 1
Stefan Behnel's avatar
Stefan Behnel committed
1439 1440
        if scope is not None:
            scope.type = self.entry.type
1441
        defined_funcs = []
1442 1443 1444 1445 1446 1447 1448
        def func_attributes(attributes):
            for attr in attributes:
                if isinstance(attr, CFuncDefNode):
                    yield attr
                elif isinstance(attr, CompilerDirectivesNode):
                    for sub_attr in func_attributes(attr.body.stats):
                        yield sub_attr
Robert Bradshaw's avatar
Robert Bradshaw committed
1449 1450 1451 1452
        if self.attributes is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for attr in self.attributes:
1453 1454 1455
                declare = getattr(attr, 'declare', None)
                if declare:
                    attr.declare(scope)
Robert Bradshaw's avatar
Robert Bradshaw committed
1456
                attr.analyse_declarations(scope)
1457 1458 1459
            for func in func_attributes(self.attributes):
                defined_funcs.append(func)
                if self.templates is not None:
1460
                    func.template_declaration = "template <typename %s>" % ", typename ".join(template_names)
1461 1462 1463 1464
        self.body = StatListNode(self.pos, stats=defined_funcs)
        self.scope = scope

    def analyse_expressions(self, env):
1465 1466
        self.body = self.body.analyse_expressions(self.entry.type.scope)
        return self
1467 1468 1469 1470 1471 1472 1473 1474 1475 1476

    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(self.entry.type.scope, code)

    def generate_execution_code(self, code):
        self.body.generate_execution_code(code)

    def annotate(self, code):
        self.body.annotate(code)

Robert Bradshaw's avatar
Robert Bradshaw committed
1477

William Stein's avatar
William Stein committed
1478 1479 1480 1481 1482
class CEnumDefNode(StatNode):
    #  name           string or None
    #  cname          string or None
    #  items          [CEnumDefItemNode]
    #  typedef_flag   boolean
1483
    #  visibility     "public" or "private" or "extern"
1484
    #  api            boolean
Stefan Behnel's avatar
Stefan Behnel committed
1485
    #  in_pxd         boolean
1486
    #  create_wrapper boolean
William Stein's avatar
William Stein committed
1487
    #  entry          Entry
Robert Bradshaw's avatar
Robert Bradshaw committed
1488

1489
    child_attrs = ["items"]
1490

1491
    def declare(self, env):
1492 1493 1494 1495
         self.entry = env.declare_enum(
             self.name, self.pos,
             cname=self.cname, typedef_flag=self.typedef_flag,
             visibility=self.visibility, api=self.api,
1496
             create_wrapper=self.create_wrapper)
1497

William Stein's avatar
William Stein committed
1498
    def analyse_declarations(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
1499 1500 1501 1502 1503
        if self.items is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for item in self.items:
                item.analyse_declarations(env, self.entry)
William Stein's avatar
William Stein committed
1504

1505
    def analyse_expressions(self, env):
1506
        return self
1507

William Stein's avatar
William Stein committed
1508
    def generate_execution_code(self, code):
1509
        if self.visibility == 'public' or self.api:
1510
            code.mark_pos(self.pos)
1511
            temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
1512 1513
            for item in self.entry.enum_values:
                code.putln("%s = PyInt_FromLong(%s); %s" % (
1514 1515 1516
                    temp,
                    item.cname,
                    code.error_goto_if_null(temp, item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1517
                code.put_gotref(temp)
1518
                code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % (
1519 1520 1521 1522
                    Naming.moddict_cname,
                    item.name,
                    temp,
                    code.error_goto(item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1523
                code.put_decref_clear(temp, PyrexTypes.py_object_type)
1524
            code.funcstate.release_temp(temp)
William Stein's avatar
William Stein committed
1525 1526 1527 1528 1529 1530


class CEnumDefItemNode(StatNode):
    #  name     string
    #  cname    string or None
    #  value    ExprNode or None
1531

1532 1533
    child_attrs = ["value"]

William Stein's avatar
William Stein committed
1534 1535
    def analyse_declarations(self, env, enum_entry):
        if self.value:
1536
            self.value = self.value.analyse_const_expression(env)
1537 1538
            if not self.value.type.is_int:
                self.value = self.value.coerce_to(PyrexTypes.c_int_type, env)
1539
                self.value = self.value.analyse_const_expression(env)
1540 1541 1542 1543 1544
        entry = env.declare_const(
            self.name, enum_entry.type,
            self.value, self.pos, cname=self.cname,
            visibility=enum_entry.visibility, api=enum_entry.api,
            create_wrapper=enum_entry.create_wrapper and enum_entry.name is None)
William Stein's avatar
William Stein committed
1545
        enum_entry.enum_values.append(entry)
1546
        if enum_entry.name:
1547
            enum_entry.type.values.append(entry.name)
William Stein's avatar
William Stein committed
1548 1549 1550


class CTypeDefNode(StatNode):
Stefan Behnel's avatar
Stefan Behnel committed
1551 1552 1553
    #  base_type    CBaseTypeNode
    #  declarator   CDeclaratorNode
    #  visibility   "public" or "private"
1554
    #  api          boolean
Stefan Behnel's avatar
Stefan Behnel committed
1555
    #  in_pxd       boolean
1556 1557

    child_attrs = ["base_type", "declarator"]
1558

William Stein's avatar
William Stein committed
1559 1560 1561 1562 1563
    def analyse_declarations(self, env):
        base = self.base_type.analyse(env)
        name_declarator, type = self.declarator.analyse(base, env)
        name = name_declarator.name
        cname = name_declarator.cname
1564

1565 1566 1567
        entry = env.declare_typedef(
            name, type, self.pos,
            cname=cname, visibility=self.visibility, api=self.api)
1568

Mark Florisson's avatar
Mark Florisson committed
1569
        if type.is_fused:
1570 1571
            entry.in_cinclude = True

Mark Florisson's avatar
Mark Florisson committed
1572 1573
        if self.in_pxd and not env.in_cinclude:
            entry.defined_in_pxd = 1
1574

William Stein's avatar
William Stein committed
1575
    def analyse_expressions(self, env):
1576 1577
        return self

William Stein's avatar
William Stein committed
1578 1579 1580 1581 1582 1583 1584 1585 1586 1587
    def generate_execution_code(self, code):
        pass


class FuncDefNode(StatNode, BlockNode):
    #  Base class for function definition nodes.
    #
    #  return_type     PyrexType
    #  #filename        string        C name of filename string const
    #  entry           Symtab.Entry
1588
    #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
Vitja Makarov's avatar
Vitja Makarov committed
1589
    #  needs_outer_scope boolean      Whether or not this function requires outer scope
1590
    #  pymethdef_required boolean     Force Python method struct generation
1591 1592
    #  directive_locals { string : ExprNode } locals defined by cython.locals(...)
    #  directive_returns [ExprNode] type defined by cython.returns(...)
1593 1594 1595 1596 1597
    #  star_arg      PyArgDeclNode or None  * argument
    #  starstar_arg  PyArgDeclNode or None  ** argument
    #
    #  is_async_def  boolean          is a Coroutine function
    #
1598 1599 1600 1601 1602
    #  has_fused_arguments  boolean
    #       Whether this cdef function has fused parameters. This is needed
    #       by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
    #       with fused argument types with a FusedCFuncDefNode

1603
    py_func = None
1604
    needs_closure = False
Vitja Makarov's avatar
Vitja Makarov committed
1605
    needs_outer_scope = False
1606
    pymethdef_required = False
1607
    is_generator = False
1608
    is_generator_body = False
1609
    is_async_def = False
Robert Bradshaw's avatar
Robert Bradshaw committed
1610
    modifiers = []
1611
    has_fused_arguments = False
1612 1613
    star_arg = None
    starstar_arg = None
1614
    is_cyfunction = False
1615
    code_object = None
1616

1617
    def analyse_default_values(self, env):
1618
        default_seen = 0
1619 1620
        for arg in self.args:
            if arg.default:
1621
                default_seen = 1
1622
                if arg.is_generic:
1623
                    arg.default = arg.default.analyse_types(env)
1624
                    arg.default = arg.default.coerce_to(arg.type, env)
1625
                else:
1626
                    error(arg.pos, "This argument cannot have a default value")
1627
                    arg.default = None
1628 1629 1630 1631
            elif arg.kw_only:
                default_seen = 1
            elif default_seen:
                error(arg.pos, "Non-default argument following default argument")
1632

1633 1634 1635 1636 1637
    def analyse_annotations(self, env):
        for arg in self.args:
            if arg.annotation:
                arg.annotation = arg.annotation.analyse_types(env)

1638
    def align_argument_type(self, env, arg):
1639
        # @cython.locals()
1640
        directive_locals = self.directive_locals
1641
        orig_type = arg.type
1642 1643 1644
        if arg.name in directive_locals:
            type_node = directive_locals[arg.name]
            other_type = type_node.analyse_as_type(env)
1645
        elif isinstance(arg, CArgDeclNode) and arg.annotation and env.directives['annotation_typing']:
1646 1647
            type_node = arg.annotation
            other_type = arg.inject_type_from_annotations(env)
1648 1649
            if other_type is None:
                return arg
1650 1651 1652 1653
        else:
            return arg
        if other_type is None:
            error(type_node.pos, "Not a type")
1654
        elif orig_type is not py_object_type and not orig_type.same_as(other_type):
1655 1656 1657 1658
            error(arg.base_type.pos, "Signature does not agree with previous declaration")
            error(type_node.pos, "Previous declaration here")
        else:
            arg.type = other_type
1659 1660
        return arg

1661 1662
    def need_gil_acquisition(self, lenv):
        return 0
1663

1664 1665
    def create_local_scope(self, env):
        genv = env
1666
        while genv.is_py_class_scope or genv.is_c_class_scope:
1667
            genv = genv.outer_scope
1668
        if self.needs_closure:
1669
            lenv = ClosureScope(name=self.entry.name,
1670 1671
                                outer_scope=genv,
                                parent_scope=env,
1672
                                scope_name=self.entry.cname)
1673
        else:
1674 1675 1676
            lenv = LocalScope(name=self.entry.name,
                              outer_scope=genv,
                              parent_scope=env)
William Stein's avatar
William Stein committed
1677
        lenv.return_type = self.return_type
1678 1679 1680
        type = self.entry.type
        if type.is_cfunction:
            lenv.nogil = type.nogil and not type.with_gil
1681
        self.local_scope = lenv
1682
        lenv.directives = env.directives
1683
        return lenv
1684

1685 1686 1687
    def generate_function_body(self, env, code):
        self.body.generate_execution_code(code)

1688
    def generate_function_definitions(self, env, code):
1689
        from . import Buffer
1690
        if self.return_type.is_memoryviewslice:
1691
            from . import MemoryView
1692 1693

        lenv = self.local_scope
Vitja Makarov's avatar
Vitja Makarov committed
1694
        if lenv.is_closure_scope and not lenv.is_passthrough:
1695 1696 1697 1698 1699
            outer_scope_cname = "%s->%s" % (Naming.cur_scope_cname,
                                            Naming.outer_scope_cname)
        else:
            outer_scope_cname = Naming.outer_scope_cname
        lenv.mangle_closure_cnames(outer_scope_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
1700 1701
        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)
Stefan Behnel's avatar
Stefan Behnel committed
1702
        # generate lambda function definitions
1703
        self.generate_lambda_definitions(lenv, code)
1704

1705 1706
        is_getbuffer_slot = (self.entry.name == "__getbuffer__" and
                             self.entry.scope.is_c_class_scope)
1707 1708 1709
        is_releasebuffer_slot = (self.entry.name == "__releasebuffer__" and
                                 self.entry.scope.is_c_class_scope)
        is_buffer_slot = is_getbuffer_slot or is_releasebuffer_slot
1710
        if is_buffer_slot:
1711 1712
            if 'cython_unused' not in self.modifiers:
                self.modifiers = self.modifiers + ['cython_unused']
1713

1714
        preprocessor_guard = self.get_preprocessor_guard()
1715

1716
        profile = code.globalstate.directives['profile']
1717 1718
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
1719 1720
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("Profile", "Profile.c"))
1721

1722
        # Generate C code for header and body of function
1723
        code.enter_cfunc_scope(lenv)
1724
        code.return_from_error_cleanup_label = code.new_label()
1725
        code.funcstate.gil_owned = not lenv.nogil
1726

William Stein's avatar
William Stein committed
1727
        # ----- Top-level constants used by this function
1728
        code.mark_pos(self.pos)
1729
        self.generate_cached_builtins_decls(lenv, code)
William Stein's avatar
William Stein committed
1730 1731
        # ----- Function header
        code.putln("")
1732 1733 1734 1735

        if preprocessor_guard:
            code.putln(preprocessor_guard)

1736 1737
        with_pymethdef = (self.needs_assignment_synthesis(env, code) or
                          self.pymethdef_required)
1738
        if self.py_func:
1739 1740 1741
            self.py_func.generate_function_header(
                code, with_pymethdef=with_pymethdef, proto_only=True)
        self.generate_function_header(code, with_pymethdef=with_pymethdef)
William Stein's avatar
William Stein committed
1742
        # ----- Local variable declarations
1743 1744 1745 1746
        # Find function scope
        cenv = env
        while cenv.is_py_class_scope or cenv.is_c_class_scope:
            cenv = cenv.outer_scope
Vitja Makarov's avatar
Vitja Makarov committed
1747
        if self.needs_closure:
1748
            code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
Robert Bradshaw's avatar
Robert Bradshaw committed
1749
            code.putln(";")
Vitja Makarov's avatar
Vitja Makarov committed
1750 1751 1752 1753
        elif self.needs_outer_scope:
            if lenv.is_passthrough:
                code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
                code.putln(";")
1754
            code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
1755
            code.putln(";")
William Stein's avatar
William Stein committed
1756
        self.generate_argument_declarations(lenv, code)
1757

1758
        for entry in lenv.var_entries:
1759
            if not (entry.in_closure or entry.is_arg):
1760
                code.put_var_declaration(entry)
1761

1762
        # Initialize the return variable __pyx_r
William Stein's avatar
William Stein committed
1763 1764
        init = ""
        if not self.return_type.is_void:
1765 1766
            if self.return_type.is_pyobject:
                init = " = NULL"
1767
            elif self.return_type.is_memoryviewslice:
1768
                init = ' = ' + MemoryView.memslice_entry_init
1769

1770 1771 1772
            code.putln("%s%s;" % (
                self.return_type.declaration_code(Naming.retval_cname),
                init))
1773

1774
        tempvardecl_code = code.insertion_point()
William Stein's avatar
William Stein committed
1775
        self.generate_keyword_list(code)
1776

William Stein's avatar
William Stein committed
1777 1778
        # ----- Extern library function declarations
        lenv.generate_library_function_declarations(code)
1779

1780
        # ----- GIL acquisition
1781
        acquire_gil = self.acquire_gil
1782

Mark Florisson's avatar
Mark Florisson committed
1783 1784
        # See if we need to acquire the GIL for variable declarations, or for
        # refnanny only
1785

1786 1787 1788
        # Closures are not currently possible for cdef nogil functions,
        # but check them anyway
        have_object_args = self.needs_closure or self.needs_outer_scope
Mark Florisson's avatar
Mark Florisson committed
1789 1790 1791 1792 1793
        for arg in lenv.arg_entries:
            if arg.type.is_pyobject:
                have_object_args = True
                break

1794 1795
        used_buffer_entries = [entry for entry in lenv.buffer_entries if entry.used]

Mark Florisson's avatar
Mark Florisson committed
1796
        acquire_gil_for_var_decls_only = (
1797 1798
            lenv.nogil and lenv.has_with_gil_block and
            (have_object_args or used_buffer_entries))
Mark Florisson's avatar
Mark Florisson committed
1799 1800

        acquire_gil_for_refnanny_only = (
1801 1802
            lenv.nogil and lenv.has_with_gil_block and not
            acquire_gil_for_var_decls_only)
Mark Florisson's avatar
Mark Florisson committed
1803 1804

        use_refnanny = not lenv.nogil or lenv.has_with_gil_block
1805 1806 1807

        if acquire_gil or acquire_gil_for_var_decls_only:
            code.put_ensure_gil()
1808
            code.funcstate.gil_owned = True
1809 1810
        elif lenv.nogil and lenv.has_with_gil_block:
            code.declare_gilstate()
1811

1812
        if profile or linetrace:
1813 1814 1815 1816 1817
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                tempvardecl_code.put_trace_declarations()
                code_object = self.code_object.calculate_result_code(code) if self.code_object else None
                code.put_trace_frame_init(code_object)
1818

1819
        # ----- set up refnanny
1820
        if use_refnanny:
1821
            tempvardecl_code.put_declare_refcount_context()
1822 1823
            code.put_setup_refcount_context(
                self.entry.name, acquire_gil=acquire_gil_for_refnanny_only)
Mark Florisson's avatar
Mark Florisson committed
1824

1825
        # ----- Automatic lead-ins for certain special functions
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1826 1827
        if is_getbuffer_slot:
            self.getbuffer_init(code)
1828
        # ----- Create closure scope object
Robert Bradshaw's avatar
Robert Bradshaw committed
1829
        if self.needs_closure:
1830
            tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__')
1831 1832 1833 1834
            slot_func_cname = TypeSlots.get_slot_function(lenv.scope_class.type.scope, tp_slot)
            if not slot_func_cname:
                slot_func_cname = '%s->tp_new' % lenv.scope_class.type.typeptr_cname
            code.putln("%s = (%s)%s(%s, %s, NULL);" % (
1835
                Naming.cur_scope_cname,
1836
                lenv.scope_class.type.empty_declaration_code(),
1837
                slot_func_cname,
1838
                lenv.scope_class.type.typeptr_cname,
1839 1840
                Naming.empty_tuple))
            code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
1841 1842 1843
            # Scope unconditionally DECREFed on return.
            code.putln("%s = %s;" % (
                Naming.cur_scope_cname,
1844 1845
                lenv.scope_class.type.cast_code("Py_None")))
            code.put_incref("Py_None", py_object_type)
1846 1847
            code.putln(code.error_goto(self.pos))
            code.putln("} else {")
Robert Bradshaw's avatar
Robert Bradshaw committed
1848
            code.put_gotref(Naming.cur_scope_cname)
1849
            code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
1850
            # Note that it is unsafe to decref the scope at this point.
Vitja Makarov's avatar
Vitja Makarov committed
1851
        if self.needs_outer_scope:
1852 1853 1854
            if self.is_cyfunction:
                code.putln("%s = (%s) __Pyx_CyFunction_GetClosure(%s);" % (
                    outer_scope_cname,
1855
                    cenv.scope_class.type.empty_declaration_code(),
1856 1857 1858 1859
                    Naming.self_cname))
            else:
                code.putln("%s = (%s) %s;" % (
                    outer_scope_cname,
1860
                    cenv.scope_class.type.empty_declaration_code(),
1861
                    Naming.self_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1862
            if lenv.is_passthrough:
Stefan Behnel's avatar
Stefan Behnel committed
1863
                code.putln("%s = %s;" % (Naming.cur_scope_cname, outer_scope_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1864
            elif self.needs_closure:
1865
                # inner closures own a reference to their outer parent
1866
                code.put_incref(outer_scope_cname, cenv.scope_class.type)
1867
                code.put_giveref(outer_scope_cname)
1868
        # ----- Trace function call
1869
        if profile or linetrace:
1870 1871
            # this looks a bit late, but if we don't get here due to a
            # fatal error before hand, it's not really worth tracing
1872 1873 1874 1875 1876 1877 1878 1879
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                if self.is_wrapper:
                    trace_name = self.entry.name + " (wrapper)"
                else:
                    trace_name = self.entry.name
                code.put_trace_call(
                    trace_name, self.pos, nogil=not code.funcstate.gil_owned)
1880
            code.funcstate.can_trace = True
William Stein's avatar
William Stein committed
1881
        # ----- Fetch arguments
1882
        self.generate_argument_parsing_code(env, code)
1883
        # If an argument is assigned to in the body, we must
Robert Bradshaw's avatar
Robert Bradshaw committed
1884
        # incref it to properly keep track of refcounts.
1885
        is_cdef = isinstance(self, CFuncDefNode)
Robert Bradshaw's avatar
Robert Bradshaw committed
1886
        for entry in lenv.arg_entries:
1887
            if entry.type.is_pyobject:
1888
                if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
1889
                    code.put_var_incref(entry)
1890

1891
            # Note: defaults are always incref-ed. For def functions, we
1892 1893 1894
            #       we aquire arguments from object converstion, so we have
            #       new references. If we are a cdef function, we need to
            #       incref our arguments
1895
            elif is_cdef and entry.type.is_memoryviewslice and len(entry.cf_assignments) > 1:
1896
                code.put_incref_memoryviewslice(entry.cname, have_gil=code.funcstate.gil_owned)
1897
        for entry in lenv.var_entries:
1898
            if entry.is_arg and len(entry.cf_assignments) > 1 and not entry.in_closure:
1899 1900 1901 1902
                if entry.xdecref_cleanup:
                    code.put_var_xincref(entry)
                else:
                    code.put_var_incref(entry)
1903

1904 1905
        # ----- Initialise local buffer auxiliary variables
        for entry in lenv.var_entries + lenv.arg_entries:
1906 1907
            if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
                Buffer.put_init_vars(entry, code)
1908

1909
        # ----- Check and convert arguments
William Stein's avatar
William Stein committed
1910
        self.generate_argument_type_tests(code)
1911 1912 1913
        # ----- Acquire buffer arguments
        for entry in lenv.arg_entries:
            if entry.type.is_buffer:
1914 1915
                Buffer.put_acquire_arg_buffer(entry, code, self.pos)

1916 1917
        if acquire_gil_for_var_decls_only:
            code.put_release_ensured_gil()
1918
            code.funcstate.gil_owned = False
1919

1920 1921 1922
        # -------------------------
        # ----- Function body -----
        # -------------------------
1923
        self.generate_function_body(env, code)
1924

1925
        code.mark_pos(self.pos, trace=False)
William Stein's avatar
William Stein committed
1926
        code.putln("")
1927 1928 1929
        code.putln("/* function exit code */")

        # ----- Default return value
1930 1931 1932 1933 1934 1935 1936 1937 1938 1939 1940
        if not self.body.is_terminator:
            if self.return_type.is_pyobject:
                #if self.return_type.is_extension_type:
                #    lhs = "(PyObject *)%s" % Naming.retval_cname
                #else:
                lhs = Naming.retval_cname
                code.put_init_to_py_none(lhs, self.return_type)
            else:
                val = self.return_type.default_value
                if val:
                    code.putln("%s = %s;" % (Naming.retval_cname, val))
1941 1942
                elif not self.return_type.is_void:
                    code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
William Stein's avatar
William Stein committed
1943
        # ----- Error cleanup
1944
        if code.error_label in code.labels_used:
1945 1946
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
1947
            code.put_label(code.error_label)
1948
            for cname, type in code.funcstate.all_managed_temps():
1949
                code.put_xdecref(cname, type, have_gil=not lenv.nogil)
1950 1951 1952

            # Clean up buffers -- this calls a Python function
            # so need to save and restore error state
1953
            buffers_present = len(used_buffer_entries) > 0
1954
            #memslice_entries = [e for e in lenv.entries.values() if e.type.is_memoryviewslice]
1955
            if buffers_present:
1956
                code.globalstate.use_utility_code(restore_exception_utility_code)
1957
                code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
1958 1959
                code.putln("__Pyx_PyThreadState_declare")
                code.putln("__Pyx_PyThreadState_assign")
1960
                code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);")
1961 1962
                for entry in used_buffer_entries:
                    Buffer.put_release_buffer_code(code, entry)
1963
                    #code.putln("%s = 0;" % entry.cname)
1964
                code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
1965

1966 1967 1968 1969 1970 1971
            if self.return_type.is_memoryviewslice:
                MemoryView.put_init_entry(Naming.retval_cname, code)
                err_val = Naming.retval_cname
            else:
                err_val = self.error_value()

1972 1973
            exc_check = self.caller_will_check_exceptions()
            if err_val is not None or exc_check:
1974
                # TODO: Fix exception tracing (though currently unused by cProfile).
Robert Bradshaw's avatar
Robert Bradshaw committed
1975 1976
                # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                # code.put_trace_exception()
1977

Mark Florisson's avatar
Mark Florisson committed
1978
                if lenv.nogil and not lenv.has_with_gil_block:
1979 1980 1981
                    code.putln("{")
                    code.put_ensure_gil()

1982
                code.put_add_traceback(self.entry.qualified_name)
1983

Mark Florisson's avatar
Mark Florisson committed
1984
                if lenv.nogil and not lenv.has_with_gil_block:
1985 1986
                    code.put_release_ensured_gil()
                    code.putln("}")
1987
            else:
1988 1989 1990
                warning(self.entry.pos,
                        "Unraisable exception in function '%s'." %
                        self.entry.qualified_name, 0)
1991
                code.put_unraisable(self.entry.qualified_name, lenv.nogil)
1992 1993 1994 1995
            default_retval = self.return_type.default_value
            if err_val is None and default_retval:
                err_val = default_retval
            if err_val is not None:
1996
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
1997 1998
            elif not self.return_type.is_void:
                code.putln("__Pyx_pretend_to_initialize(&%s);" % Naming.retval_cname)
1999 2000 2001 2002 2003 2004 2005

            if is_getbuffer_slot:
                self.getbuffer_error_cleanup(code)

            # If we are using the non-error cleanup section we should
            # jump past it if we have an error. The if-test below determine
            # whether this section is used.
2006
            if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice:
2007 2008 2009
                code.put_goto(code.return_from_error_cleanup_label)

        # ----- Non-error return cleanup
William Stein's avatar
William Stein committed
2010
        code.put_label(code.return_label)
2011 2012
        for entry in used_buffer_entries:
            Buffer.put_release_buffer_code(code, entry)
2013 2014
        if is_getbuffer_slot:
            self.getbuffer_normal_cleanup(code)
2015 2016 2017

        if self.return_type.is_memoryviewslice:
            # See if our return value is uninitialized on non-error return
2018
            # from . import MemoryView
2019
            # MemoryView.err_if_nogil_initialized_check(self.pos, env)
2020
            cond = code.unlikely(self.return_type.error_condition(Naming.retval_cname))
2021 2022 2023 2024 2025
            code.putln(
                'if (%s) {' % cond)
            if env.nogil:
                code.put_ensure_gil()
            code.putln(
2026
                'PyErr_SetString(PyExc_TypeError, "Memoryview return value is not initialized");')
2027 2028 2029 2030 2031
            if env.nogil:
                code.put_release_ensured_gil()
            code.putln(
                '}')

2032 2033
        # ----- Return cleanup for both error and no-error return
        code.put_label(code.return_from_error_cleanup_label)
2034

2035
        for entry in lenv.var_entries:
2036 2037
            if not entry.used or entry.in_closure:
                continue
2038

2039
            if entry.type.is_memoryviewslice:
2040
                code.put_xdecref_memoryviewslice(entry.cname, have_gil=not lenv.nogil)
2041
            elif entry.type.is_pyobject:
2042
                if not entry.is_arg or len(entry.cf_assignments) > 1:
2043 2044 2045 2046
                    if entry.xdecref_cleanup:
                        code.put_var_xdecref(entry)
                    else:
                        code.put_var_decref(entry)
2047

Robert Bradshaw's avatar
Robert Bradshaw committed
2048
        # Decref any increfed args
2049
        for entry in lenv.arg_entries:
2050
            if entry.type.is_pyobject:
2051
                if (acquire_gil or len(entry.cf_assignments) > 1) and not entry.in_closure:
2052
                    code.put_var_decref(entry)
2053 2054 2055 2056
            elif (entry.type.is_memoryviewslice and
                  (not is_cdef or len(entry.cf_assignments) > 1)):
                # decref slices of def functions and acquired slices from cdef
                # functions, but not borrowed slices from cdef functions.
2057 2058
                code.put_xdecref_memoryviewslice(entry.cname,
                                                 have_gil=not lenv.nogil)
Robert Bradshaw's avatar
Robert Bradshaw committed
2059 2060
        if self.needs_closure:
            code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
2061

2062
        # ----- Return
2063
        # This code is duplicated in ModuleNode.generate_module_init_func
2064 2065 2066 2067
        if not lenv.nogil:
            default_retval = self.return_type.default_value
            err_val = self.error_value()
            if err_val is None and default_retval:
Stefan Behnel's avatar
Stefan Behnel committed
2068
                err_val = default_retval  # FIXME: why is err_val not used?
2069 2070
            if self.return_type.is_pyobject:
                code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
2071

2072 2073
        if self.entry.is_special and self.entry.name == "__hash__":
            # Returning -1 for __hash__ is supposed to signal an error
2074
            # We do as Python instances and coerce -1 into -2.
2075
            code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % (
2076
                Naming.retval_cname, Naming.retval_cname))
2077

2078 2079
        if profile or linetrace:
            code.funcstate.can_trace = False
2080 2081 2082 2083 2084 2085 2086 2087
            if not self.is_generator:
                # generators are traced when iterated, not at creation
                if self.return_type.is_pyobject:
                    code.put_trace_return(
                        Naming.retval_cname, nogil=not code.funcstate.gil_owned)
                else:
                    code.put_trace_return(
                        "Py_None", nogil=not code.funcstate.gil_owned)
2088

2089
        if not lenv.nogil:
Stefan Behnel's avatar
Stefan Behnel committed
2090
            # GIL holding function
2091
            code.put_finish_refcount_context()
2092

2093 2094
        if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
            # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
2095
            code.put_release_ensured_gil()
2096
            code.funcstate.gil_owned = False
2097

William Stein's avatar
William Stein committed
2098
        if not self.return_type.is_void:
2099
            code.putln("return %s;" % Naming.retval_cname)
2100

William Stein's avatar
William Stein committed
2101
        code.putln("}")
2102 2103 2104 2105

        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

2106
        # ----- Go back and insert temp variable declarations
2107
        tempvardecl_code.put_temp_declarations(code.funcstate)
2108

2109
        # ----- Python version
2110
        code.exit_cfunc_scope()
2111
        if self.py_func:
2112
            self.py_func.generate_function_definitions(env, code)
2113
        self.generate_wrapper_functions(code)
William Stein's avatar
William Stein committed
2114 2115 2116 2117

    def declare_argument(self, env, arg):
        if arg.type.is_void:
            error(arg.pos, "Invalid use of 'void'")
2118
        elif not arg.type.is_complete() and not (arg.type.is_array or arg.type.is_memoryviewslice):
2119
            error(arg.pos, "Argument type '%s' is incomplete" % arg.type)
William Stein's avatar
William Stein committed
2120
        return env.declare_arg(arg.name, arg.type, arg.pos)
2121

2122 2123 2124
    def generate_arg_type_test(self, arg, code):
        # Generate type test for one argument.
        if arg.type.typeobj_is_available():
2125 2126
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ArgTypeTest", "FunctionArguments.c"))
2127 2128 2129 2130
            typeptr_cname = arg.type.typeptr_cname
            arg_code = "((PyObject *)%s)" % arg.entry.cname
            code.putln(
                'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % (
2131
                    arg_code,
2132 2133 2134
                    typeptr_cname,
                    arg.accept_none,
                    arg.name,
2135
                    arg.type.is_builtin_type and arg.type.require_exact,
2136 2137
                    code.error_goto(arg.pos)))
        else:
2138
            error(arg.pos, "Cannot test type of extern C class without type object name specification")
2139 2140 2141

    def generate_arg_none_check(self, arg, code):
        # Generate None check for one argument.
2142 2143 2144 2145 2146 2147
        if arg.type.is_memoryviewslice:
            cname = "%s.memview" % arg.entry.cname
        else:
            cname = arg.entry.cname

        code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname)
2148 2149
        code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % (
            max(200, len(arg.name)), arg.name,
2150 2151
            code.error_goto(arg.pos)))
        code.putln('}')
2152

2153
    def generate_wrapper_functions(self, code):
William Stein's avatar
William Stein committed
2154 2155 2156
        pass

    def generate_execution_code(self, code):
2157
        code.mark_pos(self.pos)
2158 2159
        # Evaluate and store argument default values
        for arg in self.args:
2160 2161
            if not arg.is_dynamic:
                arg.generate_assignment_code(code)
William Stein's avatar
William Stein committed
2162

2163
    #
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2164
    # Special code for the __getbuffer__ function
2165 2166 2167 2168 2169
    #
    def getbuffer_init(self, code):
        info = self.local_scope.arg_entries[1].cname
        # Python 3.0 betas have a bug in memoryview which makes it call
        # getbuffer with a NULL parameter. For now we work around this;
2170 2171
        # the following block should be removed when this bug is fixed.
        code.putln("if (%s != NULL) {" % info)
2172
        code.putln("%s->obj = Py_None; __Pyx_INCREF(Py_None);" % info)
2173
        code.put_giveref("%s->obj" % info) # Do not refnanny object within structs
2174
        code.putln("}")
2175 2176 2177

    def getbuffer_error_cleanup(self, code):
        info = self.local_scope.arg_entries[1].cname
2178 2179
        code.putln("if (%s != NULL && %s->obj != NULL) {"
                   % (info, info))
2180
        code.put_gotref("%s->obj" % info)
2181 2182 2183
        code.putln("__Pyx_DECREF(%s->obj); %s->obj = NULL;"
                   % (info, info))
        code.putln("}")
2184 2185 2186

    def getbuffer_normal_cleanup(self, code):
        info = self.local_scope.arg_entries[1].cname
2187
        code.putln("if (%s != NULL && %s->obj == Py_None) {" % (info, info))
2188 2189 2190
        code.put_gotref("Py_None")
        code.putln("__Pyx_DECREF(Py_None); %s->obj = NULL;" % info)
        code.putln("}")
William Stein's avatar
William Stein committed
2191

2192
    def get_preprocessor_guard(self):
2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203
        if not self.entry.is_special:
            return None
        name = self.entry.name
        slot = TypeSlots.method_name_to_slot.get(name)
        if not slot:
            return None
        if name == '__long__' and not self.entry.scope.lookup_here('__int__'):
            return None
        if name in ("__getbuffer__", "__releasebuffer__") and self.entry.scope.is_c_class_scope:
            return None
        return slot.preprocessor_guard_code()
2204 2205


William Stein's avatar
William Stein committed
2206 2207 2208
class CFuncDefNode(FuncDefNode):
    #  C function definition.
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
2209
    #  modifiers     ['inline']
William Stein's avatar
William Stein committed
2210 2211 2212
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarator    CDeclaratorNode
2213 2214 2215
    #  cfunc_declarator  the CFuncDeclarator of this function
    #                    (this is also available through declarator or a
    #                     base thereof)
William Stein's avatar
William Stein committed
2216
    #  body          StatListNode
2217
    #  api           boolean
2218
    #  decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2219
    #
2220
    #  with_gil      boolean    Acquire GIL around body
William Stein's avatar
William Stein committed
2221
    #  type          CFuncType
2222
    #  py_func       wrapper for calling from Python
2223
    #  overridable   whether or not this is a cpdef function
2224
    #  inline_in_pxd whether this is an inline function in a pxd file
2225
    #  template_declaration  String or None   Used for c++ class methods
Robert Bradshaw's avatar
Robert Bradshaw committed
2226
    #  is_const_method whether this is a const method
Robert Bradshaw's avatar
Robert Bradshaw committed
2227
    #  is_static_method whether this is a static method
2228
    #  is_c_class_method whether this is a cclass method
2229

2230
    child_attrs = ["base_type", "declarator", "body", "py_func_stat"]
2231 2232

    inline_in_pxd = False
2233
    decorators = None
2234
    directive_locals = None
2235
    directive_returns = None
2236
    override = None
2237
    template_declaration = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2238
    is_const_method = False
2239
    py_func_stat = None
2240

William Stein's avatar
William Stein committed
2241 2242
    def unqualified_name(self):
        return self.entry.name
2243

2244 2245 2246 2247 2248
    @property
    def code_object(self):
        # share the CodeObject with the cpdef wrapper (if available)
        return self.py_func.code_object if self.py_func else None

William Stein's avatar
William Stein committed
2249
    def analyse_declarations(self, env):
2250
        self.is_c_class_method = env.is_c_class_scope
2251 2252
        if self.directive_locals is None:
            self.directive_locals = {}
2253
        self.directive_locals.update(env.directives['locals'])
2254 2255 2256 2257 2258 2259 2260
        if self.directive_returns is not None:
            base_type = self.directive_returns.analyse_as_type(env)
            if base_type is None:
                error(self.directive_returns.pos, "Not a type")
                base_type = PyrexTypes.error_type
        else:
            base_type = self.base_type.analyse(env)
2261
        self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod')
2262
        # The 2 here is because we need both function and argument names.
2263
        if isinstance(self.declarator, CFuncDeclaratorNode):
2264 2265 2266
            name_declarator, type = self.declarator.analyse(
                base_type, env, nonempty=2 * (self.body is not None),
                directive_locals=self.directive_locals)
2267
        else:
2268 2269
            name_declarator, type = self.declarator.analyse(
                base_type, env, nonempty=2 * (self.body is not None))
2270
        if not type.is_cfunction:
2271
            error(self.pos, "Suite attached to non-function declaration")
William Stein's avatar
William Stein committed
2272 2273 2274 2275 2276
        # Remember the actual type according to the function header
        # written here, because the type in the symbol table entry
        # may be different if we're overriding a C method inherited
        # from the base type of an extension type.
        self.type = type
2277
        type.is_overridable = self.overridable
2278 2279 2280
        declarator = self.declarator
        while not hasattr(declarator, 'args'):
            declarator = declarator.base
2281 2282

        self.cfunc_declarator = declarator
2283
        self.args = declarator.args
2284

2285 2286 2287
        opt_arg_count = self.cfunc_declarator.optional_arg_count
        if (self.visibility == 'public' or self.api) and opt_arg_count:
            error(self.cfunc_declarator.pos,
2288
                  "Function with optional arguments may not be declared public or api")
2289

2290
        if type.exception_check == '+' and self.visibility != 'extern':
2291 2292 2293
            warning(self.cfunc_declarator.pos,
                    "Only extern functions can throw C++ exceptions.")

2294
        for formal_arg, type_arg in zip(self.args, type.args):
2295
            self.align_argument_type(env, type_arg)
2296
            formal_arg.type = type_arg.type
2297
            formal_arg.name = type_arg.name
2298
            formal_arg.cname = type_arg.cname
2299

2300 2301
            self._validate_type_visibility(type_arg.type, type_arg.pos, env)

2302 2303 2304
            if type_arg.type.is_fused:
                self.has_fused_arguments = True

2305 2306
            if type_arg.type.is_buffer and 'inline' in self.modifiers:
                warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
2307

2308
            if type_arg.type.is_buffer or type_arg.type.is_pythran_expr:
2309 2310
                if self.type.nogil:
                    error(formal_arg.pos,
2311
                          "Buffer may not be acquired without the GIL. Consider using memoryview slices instead.")
2312 2313 2314
                elif 'inline' in self.modifiers:
                    warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)

2315 2316
        self._validate_type_visibility(type.return_type, self.pos, env)

William Stein's avatar
William Stein committed
2317 2318
        name = name_declarator.name
        cname = name_declarator.cname
2319

Robert Bradshaw's avatar
Robert Bradshaw committed
2320
        type.is_const_method = self.is_const_method
Robert Bradshaw's avatar
Robert Bradshaw committed
2321
        type.is_static_method = self.is_static_method
William Stein's avatar
William Stein committed
2322
        self.entry = env.declare_cfunction(
2323
            name, type, self.pos,
2324 2325 2326
            cname=cname, visibility=self.visibility, api=self.api,
            defining=self.body is not None, modifiers=self.modifiers,
            overridable=self.overridable)
2327
        self.entry.inline_func_in_pxd = self.inline_in_pxd
William Stein's avatar
William Stein committed
2328
        self.return_type = type.return_type
2329
        if self.return_type.is_array and self.visibility != 'extern':
2330
            error(self.pos, "Function cannot return an array")
2331 2332
        if self.return_type.is_cpp_class:
            self.return_type.check_nullary_constructor(self.pos, "used as a return value")
2333

Robert Bradshaw's avatar
Robert Bradshaw committed
2334
        if self.overridable and not env.is_module_scope and not self.is_static_method:
2335 2336 2337
            if len(self.args) < 1 or not self.args[0].type.is_pyobject:
                # An error will be produced in the cdef function
                self.overridable = False
2338

2339 2340 2341 2342
        self.declare_cpdef_wrapper(env)
        self.create_local_scope(env)

    def declare_cpdef_wrapper(self, env):
2343
        if self.overridable:
2344 2345 2346
            if self.is_static_method:
                # TODO(robertwb): Finish this up, perhaps via more function refactoring.
                error(self.pos, "static cpdef methods not yet supported")
2347
            name = self.entry.name
2348
            py_func_body = self.call_self_node(is_module_scope=env.is_module_scope)
2349 2350 2351 2352 2353 2354
            if self.is_static_method:
                from .ExprNodes import NameNode
                decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))]
                decorators[0].decorator.analyse_types(env)
            else:
                decorators = []
2355 2356 2357 2358 2359 2360 2361 2362 2363
            self.py_func = DefNode(pos=self.pos,
                                   name=self.entry.name,
                                   args=self.args,
                                   star_arg=None,
                                   starstar_arg=None,
                                   doc=self.doc,
                                   body=py_func_body,
                                   decorators=decorators,
                                   is_wrapper=1)
2364
            self.py_func.is_module_scope = env.is_module_scope
2365
            self.py_func.analyse_declarations(env)
2366
            self.py_func.entry.is_overridable = True
2367
            self.py_func_stat = StatListNode(self.pos, stats=[self.py_func])
2368
            self.py_func.type = PyrexTypes.py_object_type
2369
            self.entry.as_variable = self.py_func.entry
2370
            self.entry.used = self.entry.as_variable.used = True
2371 2372
            # Reset scope entry the above cfunction
            env.entries[name] = self.entry
2373
            if (not self.entry.is_final_cmethod and
2374 2375
                    (not env.is_module_scope or Options.lookup_module_cpdef)):
                self.override = OverrideCheckNode(self.pos, py_func=self.py_func)
2376
                self.body = StatListNode(self.pos, stats=[self.override, self.body])
2377

2378 2379 2380 2381 2382 2383 2384 2385 2386 2387
    def _validate_type_visibility(self, type, pos, env):
        """
        Ensure that types used in cdef functions are public or api, or
        defined in a C header.
        """
        public_or_api = (self.visibility == 'public' or self.api)
        entry = getattr(type, 'entry', None)
        if public_or_api and entry and env.is_module_scope:
            if not (entry.visibility in ('public', 'extern') or
                    entry.api or entry.in_cinclude):
2388
                error(pos, "Function declared public or api may not have private types")
2389

2390
    def call_self_node(self, omit_optional_args=0, is_module_scope=0):
2391
        from . import ExprNodes
2392 2393 2394 2395
        args = self.type.args
        if omit_optional_args:
            args = args[:len(args) - self.type.optional_arg_count]
        arg_names = [arg.name for arg in args]
2396
        if is_module_scope:
2397
            cfunc = ExprNodes.NameNode(self.pos, name=self.entry.name)
2398 2399 2400 2401
            call_arg_names = arg_names
            skip_dispatch = Options.lookup_module_cpdef
        elif self.type.is_static_method:
            class_entry = self.entry.scope.parent_type.entry
2402 2403 2404
            class_node = ExprNodes.NameNode(self.pos, name=class_entry.name)
            class_node.entry = class_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=class_node, attribute=self.entry.name)
2405 2406 2407
            # Calling static c(p)def methods on an instance disallowed.
            # TODO(robertwb): Support by passing self to check for override?
            skip_dispatch = True
2408
        else:
2409 2410 2411 2412
            type_entry = self.type.args[0].type.entry
            type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
            type_arg.entry = type_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
2413
        skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
2414 2415 2416 2417 2418
        c_call = ExprNodes.SimpleCallNode(
            self.pos,
            function=cfunc,
            args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names],
            wrapper_call=skip_dispatch)
2419 2420
        return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)

William Stein's avatar
William Stein committed
2421 2422 2423 2424 2425
    def declare_arguments(self, env):
        for arg in self.type.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            self.declare_argument(env, arg)
2426

2427
    def need_gil_acquisition(self, lenv):
2428 2429
        return self.type.with_gil

2430
    def nogil_check(self, env):
2431
        type = self.type
2432
        with_gil = type.with_gil
2433 2434 2435 2436
        if type.nogil and not with_gil:
            if type.return_type.is_pyobject:
                error(self.pos,
                      "Function with Python return type cannot be declared nogil")
2437
            for entry in self.local_scope.var_entries:
2438
                if entry.type.is_pyobject and not entry.in_with_gil_block:
2439 2440
                    error(self.pos, "Function declared nogil has Python locals or temporaries")

2441
    def analyse_expressions(self, env):
2442
        self.local_scope.directives = env.directives
2443 2444 2445 2446
        if self.py_func_stat is not None:
            # this will also analyse the default values and the function name assignment
            self.py_func_stat = self.py_func_stat.analyse_expressions(env)
        elif self.py_func is not None:
2447
            # this will also analyse the default values
2448
            self.py_func = self.py_func.analyse_expressions(env)
2449 2450
        else:
            self.analyse_default_values(env)
2451
            self.analyse_annotations(env)
2452
        self.acquire_gil = self.need_gil_acquisition(self.local_scope)
2453
        return self
2454

Robert Bradshaw's avatar
Robert Bradshaw committed
2455 2456 2457
    def needs_assignment_synthesis(self, env, code=None):
        return False

2458
    def generate_function_header(self, code, with_pymethdef, with_opt_args=1, with_dispatch=1, cname=None):
2459
        scope = self.local_scope
William Stein's avatar
William Stein committed
2460 2461
        arg_decls = []
        type = self.type
2462
        for arg in type.args[:len(type.args)-type.optional_arg_count]:
2463 2464 2465 2466 2467
            arg_decl = arg.declaration_code()
            entry = scope.lookup(arg.name)
            if not entry.cf_used:
                arg_decl = 'CYTHON_UNUSED %s' % arg_decl
            arg_decls.append(arg_decl)
2468
        if with_dispatch and self.overridable:
2469 2470 2471 2472 2473 2474
            dispatch_arg = PyrexTypes.c_int_type.declaration_code(
                Naming.skip_dispatch_cname)
            if self.override:
                arg_decls.append(dispatch_arg)
            else:
                arg_decls.append('CYTHON_UNUSED %s' % dispatch_arg)
2475 2476
        if type.optional_arg_count and with_opt_args:
            arg_decls.append(type.op_arg_struct.declaration_code(Naming.optional_args_cname))
William Stein's avatar
William Stein committed
2477 2478 2479 2480
        if type.has_varargs:
            arg_decls.append("...")
        if not arg_decls:
            arg_decls = ["void"]
2481 2482
        if cname is None:
            cname = self.entry.func_cname
2483
        entity = type.function_header_code(cname, ', '.join(arg_decls))
2484
        if self.entry.visibility == 'private' and '::' not in cname:
2485
            storage_class = "static "
William Stein's avatar
William Stein committed
2486
        else:
2487
            storage_class = ""
2488
        dll_linkage = None
2489
        modifiers = code.build_function_modifiers(self.entry.func_modifiers)
Robert Bradshaw's avatar
Robert Bradshaw committed
2490

2491 2492
        header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage)
        #print (storage_class, modifiers, header)
2493
        needs_proto = self.is_c_class_method
2494
        if self.template_declaration:
2495 2496
            if needs_proto:
                code.globalstate.parts['module_declarations'].putln(self.template_declaration)
2497
            code.putln(self.template_declaration)
2498
        if needs_proto:
2499 2500
            code.globalstate.parts['module_declarations'].putln(
                "%s%s%s; /* proto*/" % (storage_class, modifiers, header))
2501
        code.putln("%s%s%s {" % (storage_class, modifiers, header))
William Stein's avatar
William Stein committed
2502 2503

    def generate_argument_declarations(self, env, code):
2504
        scope = self.local_scope
2505
        for arg in self.args:
2506
            if arg.default:
2507
                entry = scope.lookup(arg.name)
2508
                if self.override or entry.cf_used:
2509 2510 2511
                    result = arg.calculate_default_value_code(code)
                    code.putln('%s = %s;' % (
                        arg.type.declaration_code(arg.cname), result))
2512

William Stein's avatar
William Stein committed
2513 2514
    def generate_keyword_list(self, code):
        pass
2515

2516
    def generate_argument_parsing_code(self, env, code):
2517
        i = 0
2518
        used = 0
2519
        scope = self.local_scope
2520 2521
        if self.type.optional_arg_count:
            code.putln('if (%s) {' % Naming.optional_args_cname)
2522
            for arg in self.args:
2523
                if arg.default:
2524
                    entry = scope.lookup(arg.name)
2525
                    if self.override or entry.cf_used:
2526 2527 2528 2529 2530 2531 2532 2533 2534 2535
                        code.putln('if (%s->%sn > %s) {' %
                                   (Naming.optional_args_cname,
                                    Naming.pyrex_prefix, i))
                        declarator = arg.declarator
                        while not hasattr(declarator, 'name'):
                            declarator = declarator.base
                        code.putln('%s = %s->%s;' %
                                   (arg.cname, Naming.optional_args_cname,
                                    self.type.opt_arg_cname(declarator.name)))
                        used += 1
2536
                    i += 1
2537
            for _ in range(used):
2538
                code.putln('}')
2539
            code.putln('}')
2540

2541 2542 2543 2544 2545 2546 2547 2548 2549 2550
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure and not arg.default:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(scope.lookup_here(arg.name))


William Stein's avatar
William Stein committed
2551 2552
    def generate_argument_conversion_code(self, code):
        pass
2553

William Stein's avatar
William Stein committed
2554
    def generate_argument_type_tests(self, code):
2555 2556 2557 2558 2559
        # Generate type tests for args whose type in a parent
        # class is a supertype of the declared type.
        for arg in self.type.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
2560 2561
            elif arg.type.is_pyobject and not arg.accept_none:
                self.generate_arg_none_check(arg, code)
2562

2563 2564 2565 2566 2567
    def generate_execution_code(self, code):
        super(CFuncDefNode, self).generate_execution_code(code)
        if self.py_func_stat:
            self.py_func_stat.generate_execution_code(code)

William Stein's avatar
William Stein committed
2568 2569 2570 2571
    def error_value(self):
        if self.return_type.is_pyobject:
            return "0"
        else:
2572 2573
            #return None
            return self.entry.type.exception_value
2574

William Stein's avatar
William Stein committed
2575
    def caller_will_check_exceptions(self):
2576
        return self.entry.type.exception_check
2577

2578 2579
    def generate_wrapper_functions(self, code):
        # If the C signature of a function has changed, we need to generate
2580
        # wrappers to put in the slots here.
2581 2582 2583 2584 2585 2586 2587
        k = 0
        entry = self.entry
        func_type = entry.type
        while entry.prev_entry is not None:
            k += 1
            entry = entry.prev_entry
            entry.func_cname = "%s%swrap_%s" % (self.entry.func_cname, Naming.pyrex_prefix, k)
2588
            code.putln()
2589 2590 2591 2592 2593
            self.generate_function_header(
                code, 0,
                with_dispatch=entry.type.is_overridable,
                with_opt_args=entry.type.optional_arg_count,
                cname=entry.func_cname)
2594 2595 2596 2597
            if not self.return_type.is_void:
                code.put('return ')
            args = self.type.args
            arglist = [arg.cname for arg in args[:len(args)-self.type.optional_arg_count]]
2598 2599 2600 2601 2602 2603 2604 2605
            if entry.type.is_overridable:
                arglist.append(Naming.skip_dispatch_cname)
            elif func_type.is_overridable:
                arglist.append('0')
            if entry.type.optional_arg_count:
                arglist.append(Naming.optional_args_cname)
            elif func_type.optional_arg_count:
                arglist.append('NULL')
2606 2607
            code.putln('%s(%s);' % (self.entry.func_cname, ', '.join(arglist)))
            code.putln('}')
2608

William Stein's avatar
William Stein committed
2609 2610 2611 2612 2613

class PyArgDeclNode(Node):
    # Argument which must be a Python object (used
    # for * and ** arguments).
    #
2614 2615 2616
    # name        string
    # entry       Symtab.Entry
    # annotation  ExprNode or None   Py3 argument annotation
2617
    child_attrs = []
2618 2619
    is_self_arg = False
    is_type_arg = False
2620 2621 2622

    def generate_function_definitions(self, env, code):
        self.entry.generate_function_definitions(env, code)
2623

2624

2625 2626 2627
class DecoratorNode(Node):
    # A decorator
    #
2628
    # decorator    NameNode or CallNode or AttributeNode
2629 2630
    child_attrs = ['decorator']

William Stein's avatar
William Stein committed
2631 2632 2633 2634 2635

class DefNode(FuncDefNode):
    # A Python function definition.
    #
    # name          string                 the Python name of the function
Stefan Behnel's avatar
Stefan Behnel committed
2636
    # lambda_name   string                 the internal name of a lambda 'function'
2637
    # decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2638
    # args          [CArgDeclNode]         formal arguments
2639
    # doc           EncodedString or None
William Stein's avatar
William Stein committed
2640
    # body          StatListNode
2641 2642
    # return_type_annotation
    #               ExprNode or None       the Py3 return type annotation
William Stein's avatar
William Stein committed
2643 2644 2645 2646
    #
    #  The following subnode is constructed internally
    #  when the def statement is inside a Python class definition.
    #
2647 2648 2649
    #  fused_py_func        DefNode     The original fused cpdef DefNode
    #                                   (in case this is a specialization)
    #  specialized_cpdefs   [DefNode]   list of specialized cpdef DefNodes
2650
    #  py_cfunc_node  PyCFunctionNode/InnerFunctionNode   The PyCFunction to create and assign
2651 2652
    #
    # decorator_indirection IndirectionNode Used to remove __Pyx_Method_ClassMethod for fused functions
2653

2654
    child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "return_type_annotation"]
2655

2656 2657 2658
    is_staticmethod = False
    is_classmethod = False

Stefan Behnel's avatar
Stefan Behnel committed
2659
    lambda_name = None
2660
    reqd_kw_flags_cname = "0"
2661
    is_wrapper = 0
2662
    no_assignment_synthesis = 0
2663
    decorators = None
2664
    return_type_annotation = None
2665
    entry = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2666
    acquire_gil = 0
2667
    self_in_stararg = 0
2668
    py_cfunc_node = None
2669
    requires_classobj = False
2670
    defaults_struct = None # Dynamic kwrds structure name
2671
    doc = None
2672

2673 2674
    fused_py_func = False
    specialized_cpdefs = None
2675 2676 2677
    py_wrapper = None
    py_wrapper_required = True
    func_cname = None
2678

2679 2680
    defaults_getter = None

2681 2682
    def __init__(self, pos, **kwds):
        FuncDefNode.__init__(self, pos, **kwds)
2683
        k = rk = r = 0
2684 2685
        for arg in self.args:
            if arg.kw_only:
2686
                k += 1
2687
                if not arg.default:
2688 2689 2690 2691 2692 2693
                    rk += 1
            if not arg.default:
                r += 1
        self.num_kwonly_args = k
        self.num_required_kw_args = rk
        self.num_required_args = r
2694

2695
    def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, modifiers=None):
2696 2697 2698 2699
        if self.star_arg:
            error(self.star_arg.pos, "cdef function cannot have star argument")
        if self.starstar_arg:
            error(self.starstar_arg.pos, "cdef function cannot have starstar argument")
2700 2701 2702 2703
        if cfunc is None:
            cfunc_args = []
            for formal_arg in self.args:
                name_declarator, type = formal_arg.analyse(scope, nonempty=1)
2704 2705 2706 2707 2708 2709 2710 2711 2712 2713 2714 2715
                cfunc_args.append(PyrexTypes.CFuncTypeArg(name=name_declarator.name,
                                                          cname=None,
                                                          type=py_object_type,
                                                          pos=formal_arg.pos))
            cfunc_type = PyrexTypes.CFuncType(return_type=py_object_type,
                                              args=cfunc_args,
                                              has_varargs=False,
                                              exception_value=None,
                                              exception_check=False,
                                              nogil=False,
                                              with_gil=False,
                                              is_overridable=overridable)
2716
            cfunc = CVarDefNode(self.pos, type=cfunc_type)
2717
        else:
2718 2719
            if scope is None:
                scope = cfunc.scope
2720 2721 2722
            cfunc_type = cfunc.type
            if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs:
                error(self.pos, "wrong number of arguments")
Stefan Behnel's avatar
Stefan Behnel committed
2723
                error(cfunc.pos, "previous declaration here")
2724 2725
            for i, (formal_arg, type_arg) in enumerate(zip(self.args, cfunc_type.args)):
                name_declarator, type = formal_arg.analyse(scope, nonempty=1,
2726
                                                           is_self_arg=(i == 0 and scope.is_c_class_scope))
Stefan Behnel's avatar
Stefan Behnel committed
2727
                if type is None or type is PyrexTypes.py_object_type:
2728 2729
                    formal_arg.type = type_arg.type
                    formal_arg.name_declarator = name_declarator
2730
        from . import ExprNodes
2731
        if cfunc_type.exception_value is None:
2732 2733
            exception_value = None
        else:
2734 2735
            exception_value = ExprNodes.ConstNode(
                self.pos, value=cfunc_type.exception_value, type=cfunc_type.return_type)
2736
        declarator = CFuncDeclaratorNode(self.pos,
2737 2738 2739 2740 2741 2742 2743
                                         base=CNameDeclaratorNode(self.pos, name=self.name, cname=None),
                                         args=self.args,
                                         has_varargs=False,
                                         exception_check=cfunc_type.exception_check,
                                         exception_value=exception_value,
                                         with_gil=cfunc_type.with_gil,
                                         nogil=cfunc_type.nogil)
2744
        return CFuncDefNode(self.pos,
2745 2746 2747 2748 2749 2750 2751 2752 2753 2754 2755 2756 2757
                            modifiers=modifiers or [],
                            base_type=CAnalysedBaseTypeNode(self.pos, type=cfunc_type.return_type),
                            declarator=declarator,
                            body=self.body,
                            doc=self.doc,
                            overridable=cfunc_type.is_overridable,
                            type=cfunc_type,
                            with_gil=cfunc_type.with_gil,
                            nogil=cfunc_type.nogil,
                            visibility='private',
                            api=False,
                            directive_locals=getattr(cfunc, 'directive_locals', {}),
                            directive_returns=returns)
2758

2759 2760 2761 2762 2763 2764 2765 2766 2767 2768 2769
    def is_cdef_func_compatible(self):
        """Determines if the function's signature is compatible with a
        cdef function.  This can be used before calling
        .as_cfunction() to see if that will be successful.
        """
        if self.needs_closure:
            return False
        if self.star_arg or self.starstar_arg:
            return False
        return True

William Stein's avatar
William Stein committed
2770
    def analyse_declarations(self, env):
2771 2772 2773 2774 2775 2776 2777 2778 2779 2780 2781 2782 2783 2784
        if self.decorators:
            for decorator in self.decorators:
                func = decorator.decorator
                if func.is_name:
                    self.is_classmethod |= func.name == 'classmethod'
                    self.is_staticmethod |= func.name == 'staticmethod'

        if self.is_classmethod and env.lookup_here('classmethod'):
            # classmethod() was overridden - not much we can do here ...
            self.is_classmethod = False
        if self.is_staticmethod and env.lookup_here('staticmethod'):
            # staticmethod() was overridden - not much we can do here ...
            self.is_staticmethod = False

2785
        if self.name == '__new__' and env.is_py_class_scope:
Vitja Makarov's avatar
Vitja Makarov committed
2786 2787
            self.is_staticmethod = 1

2788
        self.analyse_argument_types(env)
2789 2790 2791 2792
        if self.name == '<lambda>':
            self.declare_lambda_function(env)
        else:
            self.declare_pyfunction(env)
2793

2794 2795
        self.analyse_signature(env)
        self.return_type = self.entry.signature.return_type()
2796 2797 2798 2799 2800 2801 2802
        # if a signature annotation provides a more specific return object type, use it
        if self.return_type is py_object_type and self.return_type_annotation:
            if env.directives['annotation_typing'] and not self.entry.is_special:
                _, return_type = _analyse_signature_annotation(self.return_type_annotation, env)
                if return_type and return_type.is_pyobject:
                    self.return_type = return_type

2803 2804
        self.create_local_scope(env)

2805 2806 2807 2808 2809 2810 2811 2812 2813 2814
        self.py_wrapper = DefNodeWrapper(
            self.pos,
            target=self,
            name=self.entry.name,
            args=self.args,
            star_arg=self.star_arg,
            starstar_arg=self.starstar_arg,
            return_type=self.return_type)
        self.py_wrapper.analyse_declarations(env)

2815
    def analyse_argument_types(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
2816
        self.directive_locals = env.directives['locals']
2817
        allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
2818 2819 2820 2821

        f2s = env.fused_to_specific
        env.fused_to_specific = None

William Stein's avatar
William Stein committed
2822
        for arg in self.args:
2823 2824 2825 2826
            if hasattr(arg, 'name'):
                name_declarator = None
            else:
                base_type = arg.base_type.analyse(env)
2827 2828 2829 2830 2831 2832 2833
                # If we hare in pythran mode and we got a buffer supported by
                # Pythran, we change this node to a fused type
                if has_np_pythran(env) and base_type.is_pythran_expr:
                    base_type = PyrexTypes.FusedType([
                        base_type,
                        #PyrexTypes.PythranExpr(pythran_type(self.type, "numpy_texpr")),
                        base_type.org_buffer])
2834 2835 2836
                name_declarator, type = \
                    arg.declarator.analyse(base_type, env)
                arg.name = name_declarator.name
2837
                arg.type = type
2838 2839 2840 2841

                if type.is_fused:
                    self.has_fused_arguments = True

2842
            self.align_argument_type(env, arg)
2843
            if name_declarator and name_declarator.cname:
2844
                error(self.pos, "Python function argument cannot have C name specification")
2845
            arg.type = arg.type.as_argument_type()
William Stein's avatar
William Stein committed
2846 2847 2848 2849
            arg.hdr_type = None
            arg.needs_conversion = 0
            arg.needs_type_test = 0
            arg.is_generic = 1
2850
            if arg.type.is_pyobject or arg.type.is_buffer or arg.type.is_memoryviewslice:
2851 2852 2853 2854
                if arg.or_none:
                    arg.accept_none = True
                elif arg.not_none:
                    arg.accept_none = False
2855
                elif (arg.type.is_extension_type or arg.type.is_builtin_type
2856
                        or arg.type.is_buffer or arg.type.is_memoryviewslice):
2857 2858 2859 2860 2861 2862
                    if arg.default and arg.default.constant_result is None:
                        # special case: def func(MyType obj = None)
                        arg.accept_none = True
                    else:
                        # default depends on compiler directive
                        arg.accept_none = allow_none_for_extension_args
2863 2864 2865
                else:
                    # probably just a plain 'object'
                    arg.accept_none = True
2866
            else:
2867
                arg.accept_none = True # won't be used, but must be there
2868
                if arg.not_none:
2869
                    error(arg.pos, "Only Python type arguments can have 'not None'")
2870
                if arg.or_none:
2871
                    error(arg.pos, "Only Python type arguments can have 'or None'")
2872 2873
        env.fused_to_specific = f2s

2874 2875 2876 2877 2878
        if has_np_pythran(env):
            self.np_args_idx = [i for i,a in enumerate(self.args) if a.type.is_numpy_buffer]
        else:
            self.np_args_idx = []

William Stein's avatar
William Stein committed
2879
    def analyse_signature(self, env):
2880
        if self.entry.is_special:
2881
            if self.decorators:
2882
                error(self.pos, "special functions of cdef classes cannot have decorators")
2883 2884 2885 2886
            self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg)
        elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg):
            # Use the simpler calling signature for zero- and one-argument functions.
            if self.entry.signature is TypeSlots.pyfunction_signature:
2887 2888
                if len(self.args) == 0:
                    self.entry.signature = TypeSlots.pyfunction_noargs
2889 2890 2891
                elif len(self.args) == 1:
                    if self.args[0].default is None and not self.args[0].kw_only:
                        self.entry.signature = TypeSlots.pyfunction_onearg
2892 2893 2894
            elif self.entry.signature is TypeSlots.pymethod_signature:
                if len(self.args) == 1:
                    self.entry.signature = TypeSlots.unaryfunc
2895 2896 2897
                elif len(self.args) == 2:
                    if self.args[1].default is None and not self.args[1].kw_only:
                        self.entry.signature = TypeSlots.ibinaryfunc
2898

William Stein's avatar
William Stein committed
2899 2900
        sig = self.entry.signature
        nfixed = sig.num_fixed_args()
2901 2902
        if (sig is TypeSlots.pymethod_signature and nfixed == 1
               and len(self.args) == 0 and self.star_arg):
2903 2904 2905 2906 2907 2908 2909
            # this is the only case where a diverging number of
            # arguments is not an error - when we have no explicit
            # 'self' parameter as in method(*args)
            sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used
            self.self_in_stararg = 1
            nfixed = 0

2910 2911
        if self.is_staticmethod and env.is_c_class_scope:
            nfixed = 0
2912
            self.self_in_stararg = True  # FIXME: why for staticmethods?
2913 2914 2915 2916 2917 2918

            self.entry.signature = sig = copy.copy(sig)
            sig.fixed_arg_format = "*"
            sig.is_staticmethod = True
            sig.has_generic_args = True

2919
        if ((self.is_classmethod or self.is_staticmethod) and
2920
                self.has_fused_arguments and env.is_c_class_scope):
2921 2922
            del self.decorator_indirection.stats[:]

2923 2924 2925 2926 2927 2928 2929
        for i in range(min(nfixed, len(self.args))):
            arg = self.args[i]
            arg.is_generic = 0
            if sig.is_self_arg(i) and not self.is_staticmethod:
                if self.is_classmethod:
                    arg.is_type_arg = 1
                    arg.hdr_type = arg.type = Builtin.type_type
William Stein's avatar
William Stein committed
2930
                else:
2931 2932 2933
                    arg.is_self_arg = 1
                    arg.hdr_type = arg.type = env.parent_type
                arg.needs_conversion = 0
William Stein's avatar
William Stein committed
2934
            else:
2935 2936 2937 2938 2939 2940 2941 2942 2943 2944 2945 2946 2947 2948 2949
                arg.hdr_type = sig.fixed_arg_type(i)
                if not arg.type.same_as(arg.hdr_type):
                    if arg.hdr_type.is_pyobject and arg.type.is_pyobject:
                        arg.needs_type_test = 1
                    else:
                        arg.needs_conversion = 1
            if arg.needs_conversion:
                arg.hdr_cname = Naming.arg_prefix + arg.name
            else:
                arg.hdr_cname = Naming.var_prefix + arg.name

        if nfixed > len(self.args):
            self.bad_signature()
            return
        elif nfixed < len(self.args):
William Stein's avatar
William Stein committed
2950 2951 2952
            if not sig.has_generic_args:
                self.bad_signature()
            for arg in self.args:
2953
                if arg.is_generic and (arg.type.is_extension_type or arg.type.is_builtin_type):
William Stein's avatar
William Stein committed
2954
                    arg.needs_type_test = 1
2955

William Stein's avatar
William Stein committed
2956 2957 2958 2959
    def bad_signature(self):
        sig = self.entry.signature
        expected_str = "%d" % sig.num_fixed_args()
        if sig.has_generic_args:
Stefan Behnel's avatar
Stefan Behnel committed
2960
            expected_str += " or more"
William Stein's avatar
William Stein committed
2961 2962 2963 2964 2965
        name = self.name
        if name.startswith("__") and name.endswith("__"):
            desc = "Special method"
        else:
            desc = "Method"
2966 2967
        error(self.pos, "%s %s has wrong number of arguments (%d declared, %s expected)" % (
            desc, self.name, len(self.args), expected_str))
2968

William Stein's avatar
William Stein committed
2969
    def declare_pyfunction(self, env):
2970 2971
        #print "DefNode.declare_pyfunction:", self.name, "in", env ###
        name = self.name
Stefan Behnel's avatar
Stefan Behnel committed
2972
        entry = env.lookup_here(name)
2973 2974
        if entry:
            if entry.is_final_cmethod and not env.parent_type.is_final_type:
Stefan Behnel's avatar
Stefan Behnel committed
2975
                error(self.pos, "Only final types can have final Python (def/cpdef) methods")
2976
            if entry.type.is_cfunction and not entry.is_builtin_cmethod and not self.is_wrapper:
2977
                warning(self.pos, "Overriding cdef method with def method.", 5)
2978
        entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper)
2979
        self.entry = entry
2980
        prefix = env.next_id(env.scope_prefix)
2981
        self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name
2982 2983
        if Options.docstrings:
            entry.doc = embed_position(self.pos, self.doc)
2984
            entry.doc_cname = Naming.funcdoc_prefix + prefix + name
2985
            if entry.is_special:
2986 2987
                if entry.name in TypeSlots.invisible or not entry.doc or (
                        entry.name in '__getattr__' and env.directives['fast_getattr']):
2988 2989 2990
                    entry.wrapperbase_cname = None
                else:
                    entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name
2991 2992
        else:
            entry.doc = None
2993

Stefan Behnel's avatar
Stefan Behnel committed
2994
    def declare_lambda_function(self, env):
2995
        entry = env.declare_lambda_function(self.lambda_name, self.pos)
Stefan Behnel's avatar
Stefan Behnel committed
2996 2997
        entry.doc = None
        self.entry = entry
2998
        self.entry.pyfunc_cname = entry.cname
Stefan Behnel's avatar
Stefan Behnel committed
2999

William Stein's avatar
William Stein committed
3000 3001 3002 3003 3004 3005 3006 3007 3008 3009
    def declare_arguments(self, env):
        for arg in self.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            if arg.needs_conversion:
                arg.entry = env.declare_var(arg.name, arg.type, arg.pos)
                if arg.type.is_pyobject:
                    arg.entry.init = "0"
            else:
                arg.entry = self.declare_argument(env, arg)
3010
            arg.entry.is_arg = 1
3011
            arg.entry.used = 1
William Stein's avatar
William Stein committed
3012 3013 3014 3015 3016 3017
            arg.entry.is_self_arg = arg.is_self_arg
        self.declare_python_arg(env, self.star_arg)
        self.declare_python_arg(env, self.starstar_arg)

    def declare_python_arg(self, env, arg):
        if arg:
3018
            if env.directives['infer_types'] != False:
3019 3020 3021 3022
                type = PyrexTypes.unspecified_type
            else:
                type = py_object_type
            entry = env.declare_var(arg.name, type, arg.pos)
3023
            entry.is_arg = 1
3024 3025 3026 3027
            entry.used = 1
            entry.init = "0"
            entry.xdecref_cleanup = 1
            arg.entry = entry
3028

William Stein's avatar
William Stein committed
3029
    def analyse_expressions(self, env):
3030
        self.local_scope.directives = env.directives
William Stein's avatar
William Stein committed
3031
        self.analyse_default_values(env)
3032 3033 3034
        self.analyse_annotations(env)
        if self.return_type_annotation:
            self.return_type_annotation = self.return_type_annotation.analyse_types(env)
3035

3036
        if not self.needs_assignment_synthesis(env) and self.decorators:
3037
            for decorator in self.decorators[::-1]:
3038
                decorator.decorator = decorator.decorator.analyse_expressions(env)
3039

3040
        self.py_wrapper.prepare_argument_coercion(env)
3041
        return self
3042

3043
    def needs_assignment_synthesis(self, env, code=None):
3044
        if self.is_staticmethod:
3045
            return True
3046
        if self.specialized_cpdefs or self.entry.is_fused_specialized:
3047
            return False
3048 3049
        if self.no_assignment_synthesis:
            return False
3050 3051
        if self.entry.is_special:
            return False
3052
        if self.entry.is_anonymous:
3053
            return True
3054
        if env.is_module_scope or env.is_c_class_scope:
3055
            if code is None:
3056
                return self.local_scope.directives['binding']
3057 3058 3059 3060
            else:
                return code.globalstate.directives['binding']
        return env.is_py_class_scope or env.is_closure_scope

3061 3062 3063 3064
    def error_value(self):
        return self.entry.signature.error_value

    def caller_will_check_exceptions(self):
3065
        return self.entry.signature.exception_check
3066 3067

    def generate_function_definitions(self, env, code):
3068
        if self.defaults_getter:
3069 3070
            # defaults getter must never live in class scopes, it's always a module function
            self.defaults_getter.generate_function_definitions(env.global_scope(), code)
3071

3072 3073 3074 3075 3076 3077 3078
        # Before closure cnames are mangled
        if self.py_wrapper_required:
            # func_cname might be modified by @cname
            self.py_wrapper.func_cname = self.entry.func_cname
            self.py_wrapper.generate_function_definitions(env, code)
        FuncDefNode.generate_function_definitions(self, env, code)

3079
    def generate_function_header(self, code, with_pymethdef, proto_only=0):
3080 3081 3082 3083 3084
        if proto_only:
            if self.py_wrapper_required:
                self.py_wrapper.generate_function_header(
                    code, with_pymethdef, True)
            return
William Stein's avatar
William Stein committed
3085
        arg_code_list = []
3086
        if self.entry.signature.has_dummy_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3087 3088 3089
            self_arg = 'PyObject *%s' % Naming.self_cname
            if not self.needs_outer_scope:
                self_arg = 'CYTHON_UNUSED ' + self_arg
3090 3091 3092 3093 3094 3095 3096 3097
            arg_code_list.append(self_arg)

        def arg_decl_code(arg):
            entry = arg.entry
            if entry.in_closure:
                cname = entry.original_cname
            else:
                cname = entry.cname
3098
            decl = entry.type.declaration_code(cname)
Stefan Behnel's avatar
Stefan Behnel committed
3099 3100 3101
            if not entry.cf_used:
                decl = 'CYTHON_UNUSED ' + decl
            return decl
3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123 3124 3125 3126 3127 3128 3129 3130 3131 3132 3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148 3149 3150 3151 3152 3153 3154 3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167

        for arg in self.args:
            arg_code_list.append(arg_decl_code(arg))
        if self.star_arg:
            arg_code_list.append(arg_decl_code(self.star_arg))
        if self.starstar_arg:
            arg_code_list.append(arg_decl_code(self.starstar_arg))
        arg_code = ', '.join(arg_code_list)
        dc = self.return_type.declaration_code(self.entry.pyfunc_cname)

        decls_code = code.globalstate['decls']
        preprocessor_guard = self.get_preprocessor_guard()
        if preprocessor_guard:
            decls_code.putln(preprocessor_guard)
        decls_code.putln(
            "static %s(%s); /* proto */" % (dc, arg_code))
        if preprocessor_guard:
            decls_code.putln("#endif")
        code.putln("static %s(%s) {" % (dc, arg_code))

    def generate_argument_declarations(self, env, code):
        pass

    def generate_keyword_list(self, code):
        pass

    def generate_argument_parsing_code(self, env, code):
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(arg.entry)
        for arg in self.star_arg, self.starstar_arg:
            if arg:
                put_into_closure(arg.entry)

    def generate_argument_type_tests(self, code):
        pass


class DefNodeWrapper(FuncDefNode):
    # DefNode python wrapper code generator

    defnode = None
    target = None # Target DefNode

    def __init__(self, *args, **kwargs):
        FuncDefNode.__init__(self, *args, **kwargs)
        self.num_kwonly_args = self.target.num_kwonly_args
        self.num_required_kw_args = self.target.num_required_kw_args
        self.num_required_args = self.target.num_required_args
        self.self_in_stararg = self.target.self_in_stararg
        self.signature = None

    def analyse_declarations(self, env):
        target_entry = self.target.entry
        name = self.name
        prefix = env.next_id(env.scope_prefix)
        target_entry.func_cname = Naming.pywrap_prefix + prefix + name
        target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name

        self.signature = target_entry.signature

3168 3169
        self.np_args_idx = self.target.np_args_idx

3170
    def prepare_argument_coercion(self, env):
3171 3172 3173 3174 3175 3176 3177 3178
        # This is only really required for Cython utility code at this time,
        # everything else can be done during code generation.  But we expand
        # all utility code here, simply because we cannot easily distinguish
        # different code types.
        for arg in self.args:
            if not arg.type.is_pyobject:
                if not arg.type.create_from_py_utility_code(env):
                    pass # will fail later
Robert Bradshaw's avatar
Robert Bradshaw committed
3179
            elif arg.hdr_type and not arg.hdr_type.is_pyobject:
3180 3181
                if not arg.hdr_type.create_to_py_utility_code(env):
                    pass # will fail later
3182

3183 3184 3185 3186 3187 3188 3189 3190 3191
        if self.starstar_arg and not self.starstar_arg.entry.cf_used:
            # we will set the kwargs argument to NULL instead of a new dict
            # and must therefore correct the control flow state
            entry = self.starstar_arg.entry
            entry.xdecref_cleanup = 1
            for ass in entry.cf_assignments:
                if not ass.is_arg and ass.lhs.is_name:
                    ass.lhs.cf_maybe_null = True

3192 3193 3194 3195 3196 3197 3198 3199 3200 3201 3202 3203 3204 3205 3206 3207
    def signature_has_nongeneric_args(self):
        argcount = len(self.args)
        if argcount == 0 or (
                argcount == 1 and (self.args[0].is_self_arg or
                                   self.args[0].is_type_arg)):
            return 0
        return 1

    def signature_has_generic_args(self):
        return self.signature.has_generic_args

    def generate_function_body(self, code):
        args = []
        if self.signature.has_dummy_arg:
            args.append(Naming.self_cname)
        for arg in self.args:
3208 3209 3210
            if arg.hdr_type and not (arg.type.is_memoryviewslice or
                                     arg.type.is_struct or
                                     arg.type.is_complex):
3211 3212 3213
                args.append(arg.type.cast_code(arg.entry.cname))
            else:
                args.append(arg.entry.cname)
3214 3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226
        if self.star_arg:
            args.append(self.star_arg.entry.cname)
        if self.starstar_arg:
            args.append(self.starstar_arg.entry.cname)
        args = ', '.join(args)
        if not self.return_type.is_void:
            code.put('%s = ' % Naming.retval_cname)
        code.putln('%s(%s);' % (
            self.target.entry.pyfunc_cname, args))

    def generate_function_definitions(self, env, code):
        lenv = self.target.local_scope
        # Generate C code for header and body of function
3227
        code.mark_pos(self.pos)
3228 3229 3230 3231 3232 3233
        code.putln("")
        code.putln("/* Python wrapper */")
        preprocessor_guard = self.target.get_preprocessor_guard()
        if preprocessor_guard:
            code.putln(preprocessor_guard)

3234
        code.enter_cfunc_scope(lenv)
3235 3236
        code.return_from_error_cleanup_label = code.new_label()

Vitja Makarov's avatar
Vitja Makarov committed
3237
        with_pymethdef = (self.target.needs_assignment_synthesis(env, code) or
3238 3239 3240 3241 3242 3243 3244 3245 3246 3247 3248 3249 3250 3251 3252 3253 3254 3255 3256 3257 3258 3259 3260
                          self.target.pymethdef_required)
        self.generate_function_header(code, with_pymethdef)
        self.generate_argument_declarations(lenv, code)
        tempvardecl_code = code.insertion_point()

        if self.return_type.is_pyobject:
            retval_init = ' = 0'
        else:
            retval_init = ''
        if not self.return_type.is_void:
            code.putln('%s%s;' % (
                self.return_type.declaration_code(Naming.retval_cname),
                retval_init))
        code.put_declare_refcount_context()
        code.put_setup_refcount_context('%s (wrapper)' % self.name)

        self.generate_argument_parsing_code(lenv, code)
        self.generate_argument_type_tests(code)
        self.generate_function_body(code)

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)

3261 3262 3263 3264
        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

3265 3266 3267 3268 3269 3270
        # ----- Error cleanup
        if code.error_label in code.labels_used:
            code.put_goto(code.return_label)
            code.put_label(code.error_label)
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
3271 3272 3273
            err_val = self.error_value()
            if err_val is not None:
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
3274 3275 3276 3277 3278 3279 3280 3281 3282 3283 3284 3285 3286 3287 3288 3289 3290 3291 3292

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
        for entry in lenv.var_entries:
            if entry.is_arg and entry.type.is_pyobject:
                code.put_var_decref(entry)

        code.put_finish_refcount_context()
        if not self.return_type.is_void:
            code.putln("return %s;" % Naming.retval_cname)
        code.putln('}')
        code.exit_cfunc_scope()
        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

    def generate_function_header(self, code, with_pymethdef, proto_only=0):
        arg_code_list = []
        sig = self.signature

3293
        if sig.has_dummy_arg or self.self_in_stararg:
3294 3295 3296 3297
            arg_code = "PyObject *%s" % Naming.self_cname
            if not sig.has_dummy_arg:
                arg_code = 'CYTHON_UNUSED ' + arg_code
            arg_code_list.append(arg_code)
3298

William Stein's avatar
William Stein committed
3299 3300
        for arg in self.args:
            if not arg.is_generic:
3301
                if arg.is_self_arg or arg.is_type_arg:
William Stein's avatar
William Stein committed
3302 3303
                    arg_code_list.append("PyObject *%s" % arg.hdr_cname)
                else:
3304 3305 3306 3307
                    arg_code_list.append(
                        arg.hdr_type.declaration_code(arg.hdr_cname))
        entry = self.target.entry
        if not entry.is_special and sig.method_flags() == [TypeSlots.method_noargs]:
3308
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
3309
        if entry.scope.is_c_class_scope and entry.name == "__ipow__":
Lisandro Dalcin's avatar
Lisandro Dalcin committed
3310
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
William Stein's avatar
William Stein committed
3311 3312
        if sig.has_generic_args:
            arg_code_list.append(
3313 3314
                "PyObject *%s, PyObject *%s" % (
                    Naming.args_cname, Naming.kwds_cname))
William Stein's avatar
William Stein committed
3315
        arg_code = ", ".join(arg_code_list)
3316 3317 3318 3319

        # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__'
        mf = ""
        if (entry.name in ("__getbuffer__", "__releasebuffer__")
3320
                and entry.scope.is_c_class_scope):
3321
            mf = "CYTHON_UNUSED "
3322
            with_pymethdef = False
3323

3324
        dc = self.return_type.declaration_code(entry.func_cname)
3325
        header = "static %s%s(%s)" % (mf, dc, arg_code)
William Stein's avatar
William Stein committed
3326
        code.putln("%s; /*proto*/" % header)
3327

3328
        if proto_only:
3329
            if self.target.fused_py_func:
3330 3331 3332
                # If we are the specialized version of the cpdef, we still
                # want the prototype for the "fused cpdef", in case we're
                # checking to see if our method was overridden in Python
3333
                self.target.fused_py_func.generate_function_header(
3334
                    code, with_pymethdef, proto_only=True)
3335
            return
3336

3337 3338 3339 3340
        if (Options.docstrings and entry.doc and
                not self.target.fused_py_func and
                not entry.scope.is_property_scope and
                (not entry.is_special or entry.wrapperbase_cname)):
3341
            # h_code = code.globalstate['h_code']
3342
            docstr = entry.doc
3343

Stefan Behnel's avatar
Stefan Behnel committed
3344
            if docstr.is_unicode:
3345
                docstr = docstr.as_utf8_string()
3346

William Stein's avatar
William Stein committed
3347
            code.putln(
3348
                'static char %s[] = %s;' % (
3349
                    entry.doc_cname,
3350
                    docstr.as_c_string_literal()))
3351

3352
            if entry.is_special:
3353
                code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
3354
                code.putln(
3355
                    "struct wrapperbase %s;" % entry.wrapperbase_cname)
3356
                code.putln('#endif')
3357

3358
        if with_pymethdef or self.target.fused_py_func:
William Stein's avatar
William Stein committed
3359
            code.put(
3360
                "static PyMethodDef %s = " % entry.pymethdef_cname)
3361
            code.put_pymethoddef(self.target.entry, ";", allow_skip=False)
William Stein's avatar
William Stein committed
3362 3363 3364 3365
        code.putln("%s {" % header)

    def generate_argument_declarations(self, env, code):
        for arg in self.args:
3366
            if arg.is_generic:
3367 3368
                if arg.needs_conversion:
                    code.putln("PyObject *%s = 0;" % arg.hdr_cname)
3369
                else:
3370
                    code.put_var_declaration(arg.entry)
3371 3372 3373
        for entry in env.var_entries:
            if entry.is_arg:
                code.put_var_declaration(entry)
3374

3375
    def generate_argument_parsing_code(self, env, code):
Stefan Behnel's avatar
Stefan Behnel committed
3376 3377
        # Generate fast equivalent of PyArg_ParseTuple call for
        # generic arguments, if any, including args/kwargs
3378 3379
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
Stefan Behnel's avatar
Stefan Behnel committed
3380
        end_label = code.new_label("argument_unpacking_done")
3381

3382 3383 3384
        has_kwonly_args = self.num_kwonly_args > 0
        has_star_or_kw_args = self.star_arg is not None \
            or self.starstar_arg is not None or has_kwonly_args
3385

3386
        for arg in self.args:
3387
            if not arg.type.is_pyobject:
3388
                if not arg.type.create_from_py_utility_code(env):
3389
                    pass  # will fail later
3390

3391
        if not self.signature_has_generic_args():
3392 3393
            if has_star_or_kw_args:
                error(self.pos, "This method cannot have * or keyword arguments")
3394
            self.generate_argument_conversion_code(code)
3395

3396 3397
        elif not self.signature_has_nongeneric_args():
            # func(*args) or func(**kw) or func(*args, **kw)
3398
            self.generate_stararg_copy_code(code)
3399

3400
        else:
3401
            self.generate_tuple_and_keyword_parsing_code(self.args, end_label, code)
3402

3403 3404
        code.error_label = old_error_label
        if code.label_used(our_error_label):
3405 3406
            if not code.label_used(end_label):
                code.put_goto(end_label)
3407 3408 3409 3410 3411
            code.put_label(our_error_label)
            if has_star_or_kw_args:
                self.generate_arg_decref(self.star_arg, code)
                if self.starstar_arg:
                    if self.starstar_arg.entry.xdecref_cleanup:
3412
                        code.put_var_xdecref_clear(self.starstar_arg.entry)
3413
                    else:
3414
                        code.put_var_decref_clear(self.starstar_arg.entry)
3415
            code.put_add_traceback(self.target.entry.qualified_name)
3416
            code.put_finish_refcount_context()
3417
            code.putln("return %s;" % self.error_value())
3418
        if code.label_used(end_label):
3419 3420
            code.put_label(end_label)

William Stein's avatar
William Stein committed
3421 3422
    def generate_arg_xdecref(self, arg, code):
        if arg:
3423
            code.put_var_xdecref_clear(arg.entry)
3424

3425 3426
    def generate_arg_decref(self, arg, code):
        if arg:
3427
            code.put_var_decref_clear(arg.entry)
William Stein's avatar
William Stein committed
3428

3429 3430
    def generate_stararg_copy_code(self, code):
        if not self.star_arg:
3431 3432
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3433 3434 3435
            code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" %
                       Naming.args_cname)
            code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % (
3436
                self.name, Naming.args_cname, self.error_value()))
3437
            code.putln("}")
3438

3439
        if self.starstar_arg:
3440
            if self.star_arg or not self.starstar_arg.entry.cf_used:
3441 3442 3443 3444 3445 3446
                kwarg_check = "unlikely(%s)" % Naming.kwds_cname
            else:
                kwarg_check = "%s" % Naming.kwds_cname
        else:
            kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
                Naming.kwds_cname, Naming.kwds_cname)
3447 3448
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c"))
3449
        code.putln(
3450 3451
            "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
                kwarg_check, Naming.kwds_cname, self.name,
3452
                bool(self.starstar_arg), self.error_value()))
3453

3454 3455
        if self.starstar_arg and self.starstar_arg.entry.cf_used:
            if all(ref.node.allow_null for ref in self.starstar_arg.entry.cf_references):
3456 3457
                code.putln("if (%s) {" % kwarg_check)
                code.putln("%s = PyDict_Copy(%s); if (unlikely(!%s)) return %s;" % (
3458 3459 3460 3461
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    self.starstar_arg.entry.cname,
                    self.error_value()))
3462 3463
                code.put_gotref(self.starstar_arg.entry.cname)
                code.putln("} else {")
3464
                code.putln("%s = NULL;" % (self.starstar_arg.entry.cname,))
3465 3466 3467
                code.putln("}")
                self.starstar_arg.entry.xdecref_cleanup = 1
            else:
3468
                code.put("%s = (%s) ? PyDict_Copy(%s) : PyDict_New(); " % (
3469 3470 3471
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    Naming.kwds_cname))
3472
                code.putln("if (unlikely(!%s)) return %s;" % (
3473
                    self.starstar_arg.entry.cname, self.error_value()))
3474 3475
                self.starstar_arg.entry.xdecref_cleanup = 0
                code.put_gotref(self.starstar_arg.entry.cname)
3476

3477
        if self.self_in_stararg and not self.target.is_staticmethod:
3478 3479
            # need to create a new tuple with 'self' inserted as first item
            code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % (
3480 3481 3482
                self.star_arg.entry.cname,
                Naming.args_cname,
                self.star_arg.entry.cname))
3483
            if self.starstar_arg and self.starstar_arg.entry.cf_used:
3484
                code.putln("{")
3485
                code.put_xdecref_clear(self.starstar_arg.entry.cname, py_object_type)
3486 3487 3488 3489 3490 3491 3492 3493 3494 3495 3496 3497 3498 3499 3500 3501 3502 3503 3504 3505 3506 3507
                code.putln("return %s;" % self.error_value())
                code.putln("}")
            else:
                code.putln("return %s;" % self.error_value())
            code.put_gotref(self.star_arg.entry.cname)
            code.put_incref(Naming.self_cname, py_object_type)
            code.put_giveref(Naming.self_cname)
            code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % (
                self.star_arg.entry.cname, Naming.self_cname))
            temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
            code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % (
                temp, temp, Naming.args_cname, temp))
            code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % (
                Naming.args_cname, temp))
            code.put_incref("item", py_object_type)
            code.put_giveref("item")
            code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % (
                self.star_arg.entry.cname, temp))
            code.putln("}")
            code.funcstate.release_temp(temp)
            self.star_arg.entry.xdecref_cleanup = 0
        elif self.star_arg:
3508 3509
            code.put_incref(Naming.args_cname, py_object_type)
            code.putln("%s = %s;" % (
3510 3511
                self.star_arg.entry.cname,
                Naming.args_cname))
3512 3513
            self.star_arg.entry.xdecref_cleanup = 0

3514
    def generate_tuple_and_keyword_parsing_code(self, args, success_label, code):
3515
        argtuple_error_label = code.new_label("argtuple_error")
3516

3517 3518 3519 3520 3521 3522 3523 3524 3525 3526 3527 3528 3529 3530 3531 3532 3533 3534 3535 3536
        positional_args = []
        required_kw_only_args = []
        optional_kw_only_args = []
        for arg in args:
            if arg.is_generic:
                if arg.default:
                    if not arg.is_self_arg and not arg.is_type_arg:
                        if arg.kw_only:
                            optional_kw_only_args.append(arg)
                        else:
                            positional_args.append(arg)
                elif arg.kw_only:
                    required_kw_only_args.append(arg)
                elif not arg.is_self_arg and not arg.is_type_arg:
                    positional_args.append(arg)

        # sort required kw-only args before optional ones to avoid special
        # cases in the unpacking code
        kw_only_args = required_kw_only_args + optional_kw_only_args

3537
        min_positional_args = self.num_required_args - self.num_required_kw_args
3538
        if len(args) > 0 and (args[0].is_self_arg or args[0].is_type_arg):
3539 3540
            min_positional_args -= 1
        max_positional_args = len(positional_args)
3541 3542
        has_fixed_positional_count = not self.star_arg and \
            min_positional_args == max_positional_args
3543
        has_kw_only_args = bool(kw_only_args)
3544

Stefan Behnel's avatar
Stefan Behnel committed
3545
        if self.num_required_kw_args:
3546 3547
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
Stefan Behnel's avatar
Stefan Behnel committed
3548

3549 3550 3551
        if self.starstar_arg or self.star_arg:
            self.generate_stararg_init_code(max_positional_args, code)

3552 3553
        code.putln('{')
        all_args = tuple(positional_args) + tuple(kw_only_args)
3554 3555
        code.putln("static PyObject **%s[] = {%s,0};" % (
            Naming.pykwdlist_cname,
3556 3557
            ','.join(['&%s' % code.intern_identifier(arg.name)
                      for arg in all_args])))
3558 3559 3560 3561 3562 3563 3564 3565 3566 3567

        # Before being converted and assigned to the target variables,
        # borrowed references to all unpacked argument values are
        # collected into a local PyObject* array called "values",
        # regardless if they were taken from default arguments,
        # positional arguments or keyword arguments.  Note that
        # C-typed default arguments are handled at conversion time,
        # so their array value is NULL in the end if no argument
        # was passed for them.
        self.generate_argument_values_setup_code(all_args, code)
3568

3569
        # --- optimised code when we receive keyword arguments
3570 3571 3572
        code.putln("if (%s(%s)) {" % (
            (self.num_required_kw_args > 0) and "likely" or "unlikely",
            Naming.kwds_cname))
3573 3574
        self.generate_keyword_unpacking_code(
            min_positional_args, max_positional_args,
3575 3576
            has_fixed_positional_count, has_kw_only_args,
            all_args, argtuple_error_label, code)
3577 3578

        # --- optimised code when we do not receive any keyword arguments
3579
        if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args:
3580 3581 3582 3583 3584 3585 3586
            # Python raises arg tuple related errors first, so we must
            # check the length here
            if min_positional_args == max_positional_args and not self.star_arg:
                compare = '!='
            else:
                compare = '<'
            code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % (
3587
                Naming.args_cname, compare, min_positional_args))
3588
            code.put_goto(argtuple_error_label)
3589

3590 3591 3592 3593
        if self.num_required_kw_args:
            # pure error case: keywords required but not passed
            if max_positional_args > min_positional_args and not self.star_arg:
                code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % (
3594
                    Naming.args_cname, max_positional_args))
3595 3596 3597 3598
                code.put_goto(argtuple_error_label)
            code.putln('} else {')
            for i, arg in enumerate(kw_only_args):
                if not arg.default:
3599
                    pystring_cname = code.intern_identifier(arg.name)
3600
                    # required keyword-only argument missing
3601
                    code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
3602 3603
                        self.name,
                        pystring_cname))
3604 3605
                    code.putln(code.error_goto(self.pos))
                    break
3606

3607
        else:
3608
            # optimised tuple unpacking code
3609
            code.putln('} else {')
3610 3611 3612 3613 3614
            if min_positional_args == max_positional_args:
                # parse the exact number of positional arguments from
                # the args tuple
                for i, arg in enumerate(positional_args):
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
3615
            else:
3616 3617 3618 3619 3620 3621 3622 3623
                # parse the positional arguments from the variable length
                # args tuple and reject illegal argument tuple sizes
                code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname)
                if self.star_arg:
                    code.putln('default:')
                reversed_args = list(enumerate(positional_args))[::-1]
                for i, arg in reversed_args:
                    if i >= min_positional_args-1:
Robert Bradshaw's avatar
Robert Bradshaw committed
3624 3625
                        if i != reversed_args[0][0]:
                            code.putln('CYTHON_FALLTHROUGH;')
3626
                        code.put('case %2d: ' % (i+1))
3627 3628
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
                if min_positional_args == 0:
Robert Bradshaw's avatar
Robert Bradshaw committed
3629
                    code.putln('CYTHON_FALLTHROUGH;')
3630 3631 3632 3633 3634 3635 3636 3637 3638
                    code.put('case  0: ')
                code.putln('break;')
                if self.star_arg:
                    if min_positional_args:
                        for i in range(min_positional_args-1, -1, -1):
                            code.putln('case %2d:' % i)
                        code.put_goto(argtuple_error_label)
                else:
                    code.put('default: ')
3639
                    code.put_goto(argtuple_error_label)
3640 3641
                code.putln('}')

3642
        code.putln('}') # end of the conditional unpacking blocks
3643

3644 3645 3646
        # Convert arg values to their final type and assign them.
        # Also inject non-Python default arguments, which do cannot
        # live in the values[] array.
3647 3648
        for i, arg in enumerate(all_args):
            self.generate_arg_assignment(arg, "values[%d]" % i, code)
3649

3650
        code.putln('}') # end of the whole argument unpacking block
3651 3652 3653 3654

        if code.label_used(argtuple_error_label):
            code.put_goto(success_label)
            code.put_label(argtuple_error_label)
3655 3656
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3657
            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
3658 3659 3660
                self.name, has_fixed_positional_count,
                min_positional_args, max_positional_args,
                Naming.args_cname))
3661 3662
            code.putln(code.error_goto(self.pos))

3663 3664 3665 3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684
    def generate_arg_assignment(self, arg, item, code):
        if arg.type.is_pyobject:
            # Python default arguments were already stored in 'item' at the very beginning
            if arg.is_generic:
                item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
            entry = arg.entry
            code.putln("%s = %s;" % (entry.cname, item))
        else:
            func = arg.type.from_py_function
            if func:
                if arg.default:
                    # C-typed default arguments must be handled here
                    code.putln('if (%s) {' % item)
                rhs = "%s(%s)" % (func, item)
                if arg.type.is_enum:
                    rhs = arg.type.cast_code(rhs)
                code.putln("%s = %s; %s" % (
                    arg.entry.cname,
                    rhs,
                    code.error_goto_if(arg.type.error_condition(arg.entry.cname), arg.pos)))
                if arg.default:
                    code.putln('} else {')
3685 3686 3687
                    code.putln("%s = %s;" % (
                        arg.entry.cname,
                        arg.calculate_default_value_code(code)))
3688 3689 3690 3691 3692 3693
                    if arg.type.is_memoryviewslice:
                        code.put_incref_memoryviewslice(arg.entry.cname,
                                                        have_gil=True)
                    code.putln('}')
            else:
                error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type)
3694

3695
    def generate_stararg_init_code(self, max_positional_args, code):
3696
        if self.starstar_arg:
3697
            self.starstar_arg.entry.xdecref_cleanup = 0
Stefan Behnel's avatar
Stefan Behnel committed
3698
            code.putln('%s = PyDict_New(); if (unlikely(!%s)) return %s;' % (
3699 3700 3701
                self.starstar_arg.entry.cname,
                self.starstar_arg.entry.cname,
                self.error_value()))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3702
            code.put_gotref(self.starstar_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3703 3704 3705
        if self.star_arg:
            self.star_arg.entry.xdecref_cleanup = 0
            code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % (
3706 3707
                Naming.args_cname,
                max_positional_args))
3708
            code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % (
3709 3710
                self.star_arg.entry.cname, Naming.args_cname,
                max_positional_args, Naming.args_cname))
3711
            code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3712
            if self.starstar_arg:
3713
                code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
3714 3715 3716 3717
            code.put_finish_refcount_context()
            code.putln('return %s;' % self.error_value())
            code.putln('}')
            code.put_gotref(self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3718 3719 3720 3721
            code.putln('} else {')
            code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple))
            code.put_incref(Naming.empty_tuple, py_object_type)
            code.putln('}')
3722

3723
    def generate_argument_values_setup_code(self, args, code):
3724
        max_args = len(args)
Stefan Behnel's avatar
Stefan Behnel committed
3725 3726
        # the 'values' array collects borrowed references to arguments
        # before doing any type coercion etc.
3727
        code.putln("PyObject* values[%d] = {%s};" % (
Stefan Behnel's avatar
Stefan Behnel committed
3728
            max_args, ','.join('0'*max_args)))
3729

3730
        if self.target.defaults_struct:
3731
            code.putln('%s *%s = __Pyx_CyFunction_Defaults(%s, %s);' % (
3732 3733
                self.target.defaults_struct, Naming.dynamic_args_cname,
                self.target.defaults_struct, Naming.self_cname))
3734

3735 3736
        # assign borrowed Python default values to the values array,
        # so that they can be overwritten by received arguments below
3737
        for i, arg in enumerate(args):
3738 3739 3740 3741
            if arg.default and arg.type.is_pyobject:
                default_value = arg.calculate_default_value_code(code)
                code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value)))

3742 3743 3744 3745
    def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args,
                                        has_fixed_positional_count, has_kw_only_args,
                                        all_args, argtuple_error_label, code):
        code.putln('Py_ssize_t kw_args;')
3746
        code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname)
3747
        # copy the values from the args tuple and check that it's not too long
3748
        code.putln('switch (pos_args) {')
Stefan Behnel's avatar
Stefan Behnel committed
3749 3750
        if self.star_arg:
            code.putln('default:')
3751
        for i in range(max_positional_args-1, -1, -1):
3752
            code.put('case %2d: ' % (i+1))
3753
            code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (
3754
                i, Naming.args_cname, i))
3755
            code.putln('CYTHON_FALLTHROUGH;')
3756
        code.putln('case  0: break;')
3757
        if not self.star_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3758
            code.put('default: ') # more arguments than allowed
3759
            code.put_goto(argtuple_error_label)
3760 3761
        code.putln('}')

3762 3763 3764 3765 3766 3767 3768 3769 3770
        # The code above is very often (but not always) the same as
        # the optimised non-kwargs tuple unpacking code, so we keep
        # the code block above at the very top, before the following
        # 'external' PyDict_Size() call, to make it easy for the C
        # compiler to merge the two separate tuple unpacking
        # implementations into one when they turn out to be identical.

        # If we received kwargs, fill up the positional/required
        # arguments with values from the kw dict
3771
        code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname)
3772
        if self.num_required_args or max_positional_args > 0:
Stefan Behnel's avatar
Stefan Behnel committed
3773 3774 3775 3776
            last_required_arg = -1
            for i, arg in enumerate(all_args):
                if not arg.default:
                    last_required_arg = i
3777 3778
            if last_required_arg < max_positional_args:
                last_required_arg = max_positional_args-1
Stefan Behnel's avatar
Stefan Behnel committed
3779
            if max_positional_args > 0:
3780
                code.putln('switch (pos_args) {')
3781
            for i, arg in enumerate(all_args[:last_required_arg+1]):
Stefan Behnel's avatar
Stefan Behnel committed
3782
                if max_positional_args > 0 and i <= max_positional_args:
3783 3784 3785
                    if self.star_arg and i == max_positional_args:
                        code.putln('default:')
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
3786 3787
                        if i != 0:
                            code.putln('CYTHON_FALLTHROUGH;')
3788
                        code.putln('case %2d:' % i)
3789
                pystring_cname = code.intern_identifier(arg.name)
3790
                if arg.default:
3791
                    if arg.kw_only:
3792
                        # optional kw-only args are handled separately below
3793
                        continue
3794
                    code.putln('if (kw_args > 0) {')
3795
                    # don't overwrite default argument
3796
                    code.putln('PyObject* value = PyDict_GetItem(%s, %s);' % (
3797
                        Naming.kwds_cname, pystring_cname))
3798
                    code.putln('if (value) { values[%d] = value; kw_args--; }' % i)
3799 3800
                    code.putln('}')
                else:
3801
                    code.putln('if (likely((values[%d] = PyDict_GetItem(%s, %s)) != 0)) kw_args--;' % (
3802
                        i, Naming.kwds_cname, pystring_cname))
3803 3804 3805 3806 3807 3808 3809 3810 3811 3812
                    if i < min_positional_args:
                        if i == 0:
                            # special case: we know arg 0 is missing
                            code.put('else ')
                            code.put_goto(argtuple_error_label)
                        else:
                            # print the correct number of values (args or
                            # kwargs) that were passed into positional
                            # arguments up to this point
                            code.putln('else {')
3813 3814
                            code.globalstate.use_utility_code(
                                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3815
                            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
3816 3817
                                self.name, has_fixed_positional_count,
                                min_positional_args, max_positional_args, i))
3818 3819 3820
                            code.putln(code.error_goto(self.pos))
                            code.putln('}')
                    elif arg.kw_only:
3821
                        code.putln('else {')
3822 3823
                        code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
                            self.name, pystring_cname))
3824 3825
                        code.putln(code.error_goto(self.pos))
                        code.putln('}')
Stefan Behnel's avatar
Stefan Behnel committed
3826 3827
            if max_positional_args > 0:
                code.putln('}')
3828

3829
        if has_kw_only_args:
3830
            # unpack optional keyword-only arguments separately because
3831
            # checking for interned strings in a dict is faster than iterating
3832
            self.generate_optional_kwonly_args_unpacking_code(all_args, code)
3833

3834
        code.putln('if (unlikely(kw_args > 0)) {')
3835 3836
        # non-positional/-required kw args left in dict: default args,
        # kw-only args, **kwargs or error
Stefan Behnel's avatar
Stefan Behnel committed
3837 3838 3839 3840 3841
        #
        # This is sort of a catch-all: except for checking required
        # arguments, this will always do the right thing for unpacking
        # keyword arguments, so that we can concentrate on optimising
        # common cases above.
3842 3843 3844
        if max_positional_args == 0:
            pos_arg_count = "0"
        elif self.star_arg:
3845
            code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % (
3846
                max_positional_args, max_positional_args))
Stefan Behnel's avatar
Stefan Behnel committed
3847 3848
            pos_arg_count = "used_pos_args"
        else:
3849
            pos_arg_count = "pos_args"
3850 3851
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
3852 3853 3854 3855 3856 3857 3858
        code.putln('if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % (
            Naming.kwds_cname,
            Naming.pykwdlist_cname,
            self.starstar_arg and self.starstar_arg.entry.cname or '0',
            pos_arg_count,
            self.name,
            code.error_goto(self.pos)))
3859
        code.putln('}')
3860

3861 3862 3863 3864 3865 3866 3867 3868 3869 3870 3871 3872 3873 3874 3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886 3887 3888 3889 3890 3891
    def generate_optional_kwonly_args_unpacking_code(self, all_args, code):
        optional_args = []
        first_optional_arg = -1
        for i, arg in enumerate(all_args):
            if not arg.kw_only or not arg.default:
                continue
            if not optional_args:
                first_optional_arg = i
            optional_args.append(arg.name)
        if optional_args:
            if len(optional_args) > 1:
                # if we receive more than the named kwargs, we either have **kwargs
                # (in which case we must iterate anyway) or it's an error (which we
                # also handle during iteration) => skip this part if there are more
                code.putln('if (kw_args > 0 && %s(kw_args <= %d)) {' % (
                    not self.starstar_arg and 'likely' or '',
                    len(optional_args)))
                code.putln('Py_ssize_t index;')
                # not unrolling the loop here reduces the C code overhead
                code.putln('for (index = %d; index < %d && kw_args > 0; index++) {' % (
                    first_optional_arg, first_optional_arg + len(optional_args)))
            else:
                code.putln('if (kw_args == 1) {')
                code.putln('const Py_ssize_t index = %d;' % first_optional_arg)
            code.putln('PyObject* value = PyDict_GetItem(%s, *%s[index]);' % (
                Naming.kwds_cname, Naming.pykwdlist_cname))
            code.putln('if (value) { values[index] = value; kw_args--; }')
            if len(optional_args) > 1:
                code.putln('}')
            code.putln('}')

William Stein's avatar
William Stein committed
3892
    def generate_argument_conversion_code(self, code):
3893 3894 3895
        # Generate code to convert arguments from signature type to
        # declared type, if needed.  Also copies signature arguments
        # into closure fields.
William Stein's avatar
William Stein committed
3896 3897 3898 3899 3900 3901 3902 3903 3904
        for arg in self.args:
            if arg.needs_conversion:
                self.generate_arg_conversion(arg, code)

    def generate_arg_conversion(self, arg, code):
        # Generate conversion code for one argument.
        old_type = arg.hdr_type
        new_type = arg.type
        if old_type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
3905 3906 3907 3908
            if arg.default:
                code.putln("if (%s) {" % arg.hdr_cname)
            else:
                code.putln("assert(%s); {" % arg.hdr_cname)
William Stein's avatar
William Stein committed
3909
            self.generate_arg_conversion_from_pyobject(arg, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
3910
            code.putln("}")
William Stein's avatar
William Stein committed
3911 3912 3913 3914
        elif new_type.is_pyobject:
            self.generate_arg_conversion_to_pyobject(arg, code)
        else:
            if new_type.assignable_from(old_type):
3915
                code.putln("%s = %s;" % (arg.entry.cname, arg.hdr_cname))
William Stein's avatar
William Stein committed
3916
            else:
3917
                error(arg.pos, "Cannot convert 1 argument from '%s' to '%s'" % (old_type, new_type))
3918

William Stein's avatar
William Stein committed
3919 3920 3921
    def generate_arg_conversion_from_pyobject(self, arg, code):
        new_type = arg.type
        func = new_type.from_py_function
3922
        # copied from CoerceFromPyTypeNode
William Stein's avatar
William Stein committed
3923
        if func:
3924 3925 3926 3927 3928
            lhs = arg.entry.cname
            rhs = "%s(%s)" % (func, arg.hdr_cname)
            if new_type.is_enum:
                rhs = PyrexTypes.typecast(new_type, PyrexTypes.c_long_type, rhs)
            code.putln("%s = %s; %s" % (
3929
                lhs,
3930
                rhs,
3931
                code.error_goto_if(new_type.error_condition(arg.entry.cname), arg.pos)))
William Stein's avatar
William Stein committed
3932
        else:
3933
            error(arg.pos, "Cannot convert Python object argument to type '%s'" % new_type)
3934

William Stein's avatar
William Stein committed
3935 3936 3937 3938
    def generate_arg_conversion_to_pyobject(self, arg, code):
        old_type = arg.hdr_type
        func = old_type.to_py_function
        if func:
Robert Bradshaw's avatar
Robert Bradshaw committed
3939
            code.putln("%s = %s(%s); %s" % (
William Stein's avatar
William Stein committed
3940 3941 3942
                arg.entry.cname,
                func,
                arg.hdr_cname,
Robert Bradshaw's avatar
Robert Bradshaw committed
3943
                code.error_goto_if_null(arg.entry.cname, arg.pos)))
3944
            code.put_var_gotref(arg.entry)
William Stein's avatar
William Stein committed
3945
        else:
3946
            error(arg.pos, "Cannot convert argument of type '%s' to Python object" % old_type)
William Stein's avatar
William Stein committed
3947 3948 3949 3950 3951 3952 3953 3954

    def generate_argument_type_tests(self, code):
        # Generate type tests for args whose signature
        # type is PyObject * and whose declared type is
        # a subtype thereof.
        for arg in self.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
3955 3956 3957
            elif not arg.accept_none and (arg.type.is_pyobject or
                                          arg.type.is_buffer or
                                          arg.type.is_memoryviewslice):
3958 3959
                self.generate_arg_none_check(arg, code)

William Stein's avatar
William Stein committed
3960
    def error_value(self):
3961
        return self.signature.error_value
3962

3963 3964

class GeneratorDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
3965
    # Generator function node that creates a new generator instance when called.
3966
    #
Stefan Behnel's avatar
Stefan Behnel committed
3967
    # gbody          GeneratorBodyDefNode   the function implementing the generator
3968 3969 3970
    #

    is_generator = True
3971
    is_coroutine = False
3972 3973
    needs_closure = True

Stefan Behnel's avatar
Stefan Behnel committed
3974
    child_attrs = DefNode.child_attrs + ["gbody"]
3975

3976
    def __init__(self, pos, **kwargs):
3977
        # XXX: don't actually needs a body
3978 3979
        kwargs['body'] = StatListNode(pos, stats=[], is_terminator=True)
        super(GeneratorDefNode, self).__init__(pos, **kwargs)
3980 3981 3982 3983 3984 3985 3986 3987

    def analyse_declarations(self, env):
        super(GeneratorDefNode, self).analyse_declarations(env)
        self.gbody.local_scope = self.local_scope
        self.gbody.analyse_declarations(env)

    def generate_function_body(self, env, code):
        body_cname = self.gbody.entry.func_cname
3988 3989
        name = code.intern_identifier(self.name)
        qualname = code.intern_identifier(self.qualname)
3990
        module_name = code.intern_identifier(self.module_name)
3991

3992
        code.putln('{')
3993
        code.putln('__pyx_CoroutineObject *gen = __Pyx_%s_New('
3994
                   '(__pyx_coroutine_body_t) %s, (PyObject *) %s, %s, %s, %s); %s' % (
3995
                       'Coroutine' if self.is_coroutine else 'Generator',
3996
                       body_cname, Naming.cur_scope_cname, name, qualname, module_name,
3997 3998
                       code.error_goto_if_null('gen', self.pos)))
        code.put_decref(Naming.cur_scope_cname, py_object_type)
3999
        if self.requires_classobj:
4000
            classobj_cname = 'gen->classobj'
4001 4002 4003 4004
            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                classobj_cname, Naming.self_cname))
            code.put_incref(classobj_cname, py_object_type)
            code.put_giveref(classobj_cname)
4005
        code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4006
        code.putln('return (PyObject *) gen;')
4007
        code.putln('}')
4008 4009

    def generate_function_definitions(self, env, code):
4010 4011
        env.use_utility_code(UtilityCode.load_cached(
            'Coroutine' if self.is_coroutine else 'Generator', "Coroutine.c"))
4012

4013 4014 4015 4016 4017
        self.gbody.generate_function_header(code, proto=True)
        super(GeneratorDefNode, self).generate_function_definitions(env, code)
        self.gbody.generate_function_definitions(env, code)


4018 4019 4020 4021
class AsyncDefNode(GeneratorDefNode):
    is_coroutine = True


4022
class GeneratorBodyDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
4023
    # Main code body of a generator implemented as a DefNode.
4024 4025 4026
    #

    is_generator_body = True
4027
    is_inlined = False
4028
    inlined_comprehension_type = None  # container type for inlined comprehensions
4029

4030
    def __init__(self, pos=None, name=None, body=None):
4031 4032 4033
        super(GeneratorBodyDefNode, self).__init__(
            pos=pos, body=body, name=name, doc=None,
            args=[], star_arg=None, starstar_arg=None)
4034

4035 4036
    def declare_generator_body(self, env):
        prefix = env.next_id(env.scope_prefix)
Vitja Makarov's avatar
Vitja Makarov committed
4037
        name = env.next_id('generator')
4038 4039 4040 4041
        cname = Naming.genbody_prefix + prefix + name
        entry = env.declare_var(None, py_object_type, self.pos,
                                cname=cname, visibility='private')
        entry.func_cname = cname
4042 4043 4044 4045 4046 4047
        entry.qualified_name = EncodedString(self.name)
        self.entry = entry

    def analyse_declarations(self, env):
        self.analyse_argument_types(env)
        self.declare_generator_body(env)
4048 4049

    def generate_function_header(self, code, proto=False):
4050
        header = "static PyObject *%s(__pyx_CoroutineObject *%s, PyObject *%s)" % (
4051
            self.entry.func_cname,
4052
            Naming.generator_cname,
4053 4054 4055 4056
            Naming.sent_value_cname)
        if proto:
            code.putln('%s; /* proto */' % header)
        else:
Stefan Behnel's avatar
Stefan Behnel committed
4057
            code.putln('%s /* generator body */\n{' % header)
4058 4059 4060 4061 4062 4063 4064 4065

    def generate_function_definitions(self, env, code):
        lenv = self.local_scope

        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)

        # Generate C code for header and body of function
4066
        code.enter_cfunc_scope(lenv)
4067 4068 4069 4070 4071 4072 4073 4074
        code.return_from_error_cleanup_label = code.new_label()

        # ----- Top-level constants used by this function
        code.mark_pos(self.pos)
        self.generate_cached_builtins_decls(lenv, code)
        # ----- Function header
        code.putln("")
        self.generate_function_header(code)
4075
        closure_init_code = code.insertion_point()
4076 4077 4078 4079 4080
        # ----- Local variables
        code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
        tempvardecl_code = code.insertion_point()
        code.put_declare_refcount_context()
        code.put_setup_refcount_context(self.entry.name)
4081 4082 4083
        profile = code.globalstate.directives['profile']
        linetrace = code.globalstate.directives['linetrace']
        if profile or linetrace:
4084
            tempvardecl_code.put_trace_declarations()
4085 4086 4087 4088 4089 4090 4091 4092 4093 4094

        # ----- Resume switch point.
        code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
        resume_code = code.insertion_point()
        first_run_label = code.new_label('first_run')
        code.use_label(first_run_label)
        code.put_label(first_run_label)
        code.putln('%s' %
                   (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))

4095 4096 4097 4098 4099 4100 4101 4102 4103 4104 4105 4106 4107 4108 4109 4110 4111
        # ----- prepare target container for inlined comprehension
        if self.is_inlined and self.inlined_comprehension_type is not None:
            target_type = self.inlined_comprehension_type
            if target_type is Builtin.list_type:
                comp_init = 'PyList_New(0)'
            elif target_type is Builtin.set_type:
                comp_init = 'PySet_New(NULL)'
            elif target_type is Builtin.dict_type:
                comp_init = 'PyDict_New()'
            else:
                raise InternalError(
                    "invalid type of inlined comprehension: %s" % target_type)
            code.putln("%s = %s; %s" % (
                Naming.retval_cname, comp_init,
                code.error_goto_if_null(Naming.retval_cname, self.pos)))
            code.put_gotref(Naming.retval_cname)

4112 4113
        # ----- Function body
        self.generate_function_body(env, code)
4114 4115 4116 4117 4118 4119
        # ----- Closure initialization
        if lenv.scope_class.type.scope.entries:
            closure_init_code.putln('%s = %s;' % (
                lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
                lenv.scope_class.type.cast_code('%s->closure' %
                                                Naming.generator_cname)))
4120
            # FIXME: this silences a potential "unused" warning => try to avoid unused closures in more cases
4121
            code.putln("CYTHON_MAYBE_UNUSED_VAR(%s);" % Naming.cur_scope_cname)
4122 4123 4124 4125 4126

        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

Stefan Behnel's avatar
Stefan Behnel committed
4127 4128
        # on normal generator termination, we do not take the exception propagation
        # path: no traceback info is required and not creating it is much faster
4129
        if not self.is_inlined and not self.body.is_terminator:
4130
            code.putln('PyErr_SetNone(PyExc_StopIteration);')
4131 4132
        # ----- Error cleanup
        if code.error_label in code.labels_used:
4133 4134
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
4135
            code.put_label(code.error_label)
4136 4137
            if self.is_inlined and self.inlined_comprehension_type is not None:
                code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4138 4139
            if Future.generator_stop in env.global_scope().context.future_directives:
                # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
4140
                code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Coroutine.c"))
4141
                code.putln("if (unlikely(PyErr_ExceptionMatches(PyExc_StopIteration))) "
4142
                           "__Pyx_Generator_Replace_StopIteration();")
4143 4144
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
4145
            code.put_add_traceback(self.entry.qualified_name)
4146 4147 4148

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
4149 4150 4151 4152
        if self.is_inlined:
            code.put_xgiveref(Naming.retval_cname)
        else:
            code.put_xdecref_clear(Naming.retval_cname, py_object_type)
4153
        code.putln('%s->resume_label = -1;' % Naming.generator_cname)
4154
        # clean up as early as possible to help breaking any reference cycles
4155
        code.putln('__Pyx_Coroutine_clear((PyObject*)%s);' % Naming.generator_cname)
4156 4157 4158
        if profile or linetrace:
            code.put_trace_return(Naming.retval_cname,
                                  nogil=not code.funcstate.gil_owned)
4159
        code.put_finish_refcount_context()
4160
        code.putln("return %s;" % Naming.retval_cname)
4161 4162 4163 4164 4165
        code.putln("}")

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)
        # ----- Generator resume code
4166 4167 4168
        if profile or linetrace:
            resume_code.put_trace_call(self.entry.qualified_name, self.pos,
                                       nogil=not code.funcstate.gil_owned)
4169
        resume_code.putln("switch (%s->resume_label) {" % (
Stefan Behnel's avatar
Stefan Behnel committed
4170
                       Naming.generator_cname))
4171

4172
        resume_code.putln("case 0: goto %s;" % first_run_label)
4173

4174 4175
        for i, label in code.yield_labels:
            resume_code.putln("case %d: goto %s;" % (i, label))
Stefan Behnel's avatar
Stefan Behnel committed
4176
        resume_code.putln("default: /* CPython raises the right error here */")
4177 4178 4179
        if profile or linetrace:
            resume_code.put_trace_return("Py_None",
                                         nogil=not code.funcstate.gil_owned)
4180
        resume_code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4181 4182
        resume_code.putln("return NULL;")
        resume_code.putln("}")
4183 4184 4185 4186

        code.exit_cfunc_scope()


4187 4188
class OverrideCheckNode(StatNode):
    # A Node for dispatching to the def method if it
4189
    # is overriden.
4190 4191 4192 4193 4194 4195
    #
    #  py_func
    #
    #  args
    #  func_temp
    #  body
4196

Robert Bradshaw's avatar
Robert Bradshaw committed
4197
    child_attrs = ['body']
4198

4199
    body = None
Robert Bradshaw's avatar
Robert Bradshaw committed
4200

4201 4202
    def analyse_expressions(self, env):
        self.args = env.arg_entries
4203 4204 4205 4206
        if self.py_func.is_module_scope:
            first_arg = 0
        else:
            first_arg = 1
4207
        from . import ExprNodes
4208
        self.func_node = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
Stefan Behnel's avatar
Stefan Behnel committed
4209 4210
        call_node = ExprNodes.SimpleCallNode(
            self.pos, function=self.func_node,
4211 4212
            args=[ExprNodes.NameNode(self.pos, name=arg.name)
                  for arg in self.args[first_arg:]])
4213 4214
        if env.return_type.is_void or env.return_type.is_returncode:
            self.body = StatListNode(self.pos, stats=[
4215 4216
                ExprStatNode(self.pos, expr=call_node),
                ReturnStatNode(self.pos, value=None)])
4217 4218
        else:
            self.body = ReturnStatNode(self.pos, value=call_node)
4219 4220
        self.body = self.body.analyse_expressions(env)
        return self
4221

4222
    def generate_execution_code(self, code):
4223
        interned_attr_cname = code.intern_identifier(self.py_func.entry.name)
4224
        # Check to see if we are an extension type
4225 4226 4227 4228
        if self.py_func.is_module_scope:
            self_arg = "((PyObject *)%s)" % Naming.module_cname
        else:
            self_arg = "((PyObject *)%s)" % self.args[0].cname
4229
        code.putln("/* Check if called by wrapper */")
4230
        code.putln("if (unlikely(%s)) ;" % Naming.skip_dispatch_cname)
Stefan Behnel's avatar
Stefan Behnel committed
4231
        code.putln("/* Check if overridden in Python */")
4232 4233 4234
        if self.py_func.is_module_scope:
            code.putln("else {")
        else:
4235
            code.putln("else if (unlikely(Py_TYPE(%s)->tp_dictoffset != 0)) {" % self_arg)
4236 4237
        func_node_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.func_node.set_cname(func_node_temp)
4238
        # need to get attribute manually--scope would return cdef method
4239 4240
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("PyObjectGetAttrStr", "ObjectHandling.c"))
4241
        err = code.error_goto_if_null(func_node_temp, self.pos)
4242
        code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % (
4243 4244 4245
            func_node_temp, self_arg, interned_attr_cname, err))
        code.put_gotref(func_node_temp)
        is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp
4246
        is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)%s)" % (
4247
            func_node_temp, self.py_func.entry.func_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
4248
        code.putln("if (!%s || %s) {" % (is_builtin_function_or_method, is_overridden))
4249 4250
        self.body.generate_execution_code(code)
        code.putln("}")
4251 4252
        code.put_decref_clear(func_node_temp, PyrexTypes.py_object_type)
        code.funcstate.release_temp(func_node_temp)
Robert Bradshaw's avatar
Robert Bradshaw committed
4253
        code.putln("}")
4254

4255

Robert Bradshaw's avatar
Robert Bradshaw committed
4256 4257
class ClassDefNode(StatNode, BlockNode):
    pass
4258

4259

Robert Bradshaw's avatar
Robert Bradshaw committed
4260
class PyClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4261 4262
    #  A Python class definition.
    #
Stefan Behnel's avatar
Stefan Behnel committed
4263
    #  name     EncodedString   Name of the class
William Stein's avatar
William Stein committed
4264 4265 4266 4267
    #  doc      string or None
    #  body     StatNode        Attribute definition code
    #  entry    Symtab.Entry
    #  scope    PyClassScope
4268
    #  decorators    [DecoratorNode]        list of decorators or None
William Stein's avatar
William Stein committed
4269 4270 4271
    #
    #  The following subnodes are constructed internally:
    #
4272
    #  dict     DictNode   Class dictionary or Py3 namespace
William Stein's avatar
William Stein committed
4273 4274
    #  classobj ClassNode  Class object
    #  target   NameNode   Variable to assign class object to
4275

4276
    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result",
4277
                   "target", "class_cell", "decorators"]
4278
    decorators = None
4279
    class_result = None
4280 4281 4282
    is_py3_style_class = False  # Python3 style class (kwargs)
    metaclass = None
    mkw = None
4283

4284
    def __init__(self, pos, name, bases, doc, body, decorators=None,
4285
                 keyword_args=None, force_py3_semantics=False):
William Stein's avatar
William Stein committed
4286 4287 4288 4289
        StatNode.__init__(self, pos)
        self.name = name
        self.doc = doc
        self.body = body
4290
        self.decorators = decorators
4291
        self.bases = bases
4292
        from . import ExprNodes
4293
        if self.doc and Options.docstrings:
4294
            doc = embed_position(self.pos, self.doc)
4295
            doc_node = ExprNodes.StringNode(pos, value=doc)
William Stein's avatar
William Stein committed
4296 4297
        else:
            doc_node = None
4298 4299

        allow_py2_metaclass = not force_py3_semantics
4300
        if keyword_args:
4301 4302
            allow_py2_metaclass = False
            self.is_py3_style_class = True
4303 4304 4305 4306 4307 4308 4309 4310 4311 4312 4313 4314 4315 4316
            if keyword_args.is_dict_literal:
                if keyword_args.key_value_pairs:
                    for i, item in list(enumerate(keyword_args.key_value_pairs))[::-1]:
                        if item.key.value == 'metaclass':
                            if self.metaclass is not None:
                                error(item.pos, "keyword argument 'metaclass' passed multiple times")
                            # special case: we already know the metaclass,
                            # so we don't need to do the "build kwargs,
                            # find metaclass" dance at runtime
                            self.metaclass = item.value
                            del keyword_args.key_value_pairs[i]
                    self.mkw = keyword_args
                else:
                    assert self.metaclass is not None
4317
            else:
4318
                # MergedDictNode
4319
                self.mkw = ExprNodes.ProxyNode(keyword_args)
4320 4321

        if force_py3_semantics or self.bases or self.mkw or self.metaclass:
4322
            if self.metaclass is None:
4323
                if keyword_args and not keyword_args.is_dict_literal:
4324 4325 4326 4327
                    # **kwargs may contain 'metaclass' arg
                    mkdict = self.mkw
                else:
                    mkdict = None
4328 4329 4330 4331 4332 4333 4334
                if (not mkdict and
                        self.bases.is_sequence_constructor and
                        not self.bases.args):
                    pass  # no base classes => no inherited metaclass
                else:
                    self.metaclass = ExprNodes.PyClassMetaclassNode(
                        pos, mkw=mkdict, bases=self.bases)
4335 4336 4337 4338 4339 4340 4341 4342 4343 4344 4345 4346 4347
                needs_metaclass_calculation = False
            else:
                needs_metaclass_calculation = True

            self.dict = ExprNodes.PyClassNamespaceNode(
                pos, name=name, doc=doc_node,
                metaclass=self.metaclass, bases=self.bases, mkw=self.mkw)
            self.classobj = ExprNodes.Py3ClassNode(
                pos, name=name,
                bases=self.bases, dict=self.dict, doc=doc_node,
                metaclass=self.metaclass, mkw=self.mkw,
                calculate_metaclass=needs_metaclass_calculation,
                allow_py2_metaclass=allow_py2_metaclass)
4348
        else:
4349 4350 4351 4352 4353 4354 4355
            # no bases, no metaclass => old style class creation
            self.dict = ExprNodes.DictNode(pos, key_value_pairs=[])
            self.classobj = ExprNodes.ClassNode(
                pos, name=name,
                bases=bases, dict=self.dict, doc=doc_node)

        self.target = ExprNodes.NameNode(pos, name=name)
4356
        self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
4357

4358 4359
    def as_cclass(self):
        """
4360
        Return this node as if it were declared as an extension class
4361
        """
4362
        if self.is_py3_style_class:
4363 4364
            error(self.classobj.pos, "Python3 style class could not be represented as C class")
            return
4365 4366 4367 4368 4369 4370 4371
        bases = self.classobj.bases.args
        if len(bases) == 0:
            base_class_name = None
            base_class_module = None
        elif len(bases) == 1:
            base = bases[0]
            path = []
4372
            from .ExprNodes import AttributeNode, NameNode
4373
            while isinstance(base, AttributeNode):
4374 4375
                path.insert(0, base.attribute)
                base = base.obj
4376
            if isinstance(base, NameNode):
4377 4378 4379 4380 4381 4382 4383 4384 4385 4386 4387
                path.insert(0, base.name)
                base_class_name = path[-1]
                if len(path) > 1:
                    base_class_module = u'.'.join(path[:-1])
                else:
                    base_class_module = None
            else:
                error(self.classobj.bases.args.pos, "Invalid base class")
        else:
            error(self.classobj.bases.args.pos, "C class may only have one base class")
            return None
4388 4389

        return CClassDefNode(self.pos,
4390 4391 4392 4393 4394 4395 4396 4397 4398
                             visibility='private',
                             module_name=None,
                             class_name=self.name,
                             base_class_module=base_class_module,
                             base_class_name=base_class_name,
                             decorators=self.decorators,
                             body=self.body,
                             in_pxd=False,
                             doc=self.doc)
4399

4400 4401
    def create_scope(self, env):
        genv = env
4402 4403
        while genv.is_py_class_scope or genv.is_c_class_scope:
            genv = genv.outer_scope
4404
        cenv = self.scope = PyClassScope(name=self.name, outer_scope=genv)
4405
        return cenv
4406

William Stein's avatar
William Stein committed
4407
    def analyse_declarations(self, env):
4408 4409
        class_result = self.classobj
        if self.decorators:
4410
            from .ExprNodes import SimpleCallNode
4411 4412 4413
            for decorator in self.decorators[::-1]:
                class_result = SimpleCallNode(
                    decorator.pos,
4414 4415
                    function=decorator.decorator,
                    args=[class_result])
4416
            self.decorators = None
4417 4418
        self.class_result = class_result
        self.class_result.analyse_declarations(env)
William Stein's avatar
William Stein committed
4419
        self.target.analyse_target_declaration(env)
4420
        cenv = self.create_scope(env)
4421
        cenv.directives = env.directives
4422 4423
        cenv.class_obj_cname = self.target.entry.cname
        self.body.analyse_declarations(cenv)
4424

William Stein's avatar
William Stein committed
4425
    def analyse_expressions(self, env):
4426
        if self.bases:
4427
            self.bases = self.bases.analyse_expressions(env)
4428
        if self.metaclass:
4429
            self.metaclass = self.metaclass.analyse_expressions(env)
4430
        if self.mkw:
4431 4432 4433
            self.mkw = self.mkw.analyse_expressions(env)
        self.dict = self.dict.analyse_expressions(env)
        self.class_result = self.class_result.analyse_expressions(env)
4434
        cenv = self.scope
4435
        self.body = self.body.analyse_expressions(cenv)
4436
        self.target.analyse_target_expression(env, self.classobj)
4437 4438
        self.class_cell = self.class_cell.analyse_expressions(cenv)
        return self
4439

4440
    def generate_function_definitions(self, env, code):
4441
        self.generate_lambda_definitions(self.scope, code)
4442
        self.body.generate_function_definitions(self.scope, code)
4443

William Stein's avatar
William Stein committed
4444
    def generate_execution_code(self, code):
4445
        code.mark_pos(self.pos)
4446 4447
        code.pyclass_stack.append(self)
        cenv = self.scope
4448
        if self.bases:
4449
            self.bases.generate_evaluation_code(code)
4450
        if self.mkw:
4451
            self.mkw.generate_evaluation_code(code)
4452
        if self.metaclass:
4453
            self.metaclass.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
4454
        self.dict.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4455
        cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
4456
        self.class_cell.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4457
        self.body.generate_execution_code(code)
4458
        self.class_result.generate_evaluation_code(code)
4459 4460 4461
        self.class_cell.generate_injection_code(
            code, self.class_result.result())
        self.class_cell.generate_disposal_code(code)
4462
        cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
4463
        self.target.generate_assignment_code(self.class_result, code)
William Stein's avatar
William Stein committed
4464
        self.dict.generate_disposal_code(code)
4465
        self.dict.free_temps(code)
4466
        if self.metaclass:
4467 4468
            self.metaclass.generate_disposal_code(code)
            self.metaclass.free_temps(code)
4469 4470 4471 4472
        if self.mkw:
            self.mkw.generate_disposal_code(code)
            self.mkw.free_temps(code)
        if self.bases:
4473 4474
            self.bases.generate_disposal_code(code)
            self.bases.free_temps(code)
4475
        code.pyclass_stack.pop()
William Stein's avatar
William Stein committed
4476

Robert Bradshaw's avatar
Robert Bradshaw committed
4477
class CClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4478 4479 4480 4481
    #  An extension type definition.
    #
    #  visibility         'private' or 'public' or 'extern'
    #  typedef_flag       boolean
Stefan Behnel's avatar
Stefan Behnel committed
4482
    #  api                boolean
William Stein's avatar
William Stein committed
4483 4484 4485 4486 4487 4488 4489 4490
    #  module_name        string or None    For import of extern type objects
    #  class_name         string            Unqualified name of class
    #  as_name            string or None    Name to declare as in this scope
    #  base_class_module  string or None    Module containing the base class
    #  base_class_name    string or None    Name of the base class
    #  objstruct_name     string or None    Specified C name of object struct
    #  typeobj_name       string or None    Specified C name of type object
    #  in_pxd             boolean           Is in a .pxd file
4491
    #  decorators         [DecoratorNode]   list of decorators or None
William Stein's avatar
William Stein committed
4492 4493 4494 4495
    #  doc                string or None
    #  body               StatNode or None
    #  entry              Symtab.Entry
    #  base_type          PyExtensionType or None
4496 4497
    #  buffer_defaults_node DictNode or None Declares defaults for a buffer
    #  buffer_defaults_pos
4498

4499
    child_attrs = ["body"]
4500 4501
    buffer_defaults_node = None
    buffer_defaults_pos = None
4502 4503 4504 4505
    typedef_flag = False
    api = False
    objstruct_name = None
    typeobj_name = None
4506
    decorators = None
4507
    shadow = False
4508

Robert Bradshaw's avatar
Robert Bradshaw committed
4509 4510
    def buffer_defaults(self, env):
        if not hasattr(self, '_buffer_defaults'):
4511
            from . import Buffer
Robert Bradshaw's avatar
Robert Bradshaw committed
4512 4513 4514 4515 4516 4517 4518 4519 4520
            if self.buffer_defaults_node:
                self._buffer_defaults = Buffer.analyse_buffer_options(
                    self.buffer_defaults_pos,
                    env, [], self.buffer_defaults_node,
                    need_complete=False)
            else:
                self._buffer_defaults = None
        return self._buffer_defaults

4521 4522 4523 4524 4525 4526 4527 4528
    def declare(self, env):
        if self.module_name and self.visibility != 'extern':
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return None
        else:
            home_scope = env
4529

4530
        self.entry = home_scope.declare_c_class(
4531 4532 4533 4534 4535 4536 4537 4538 4539 4540 4541 4542 4543
            name=self.class_name,
            pos=self.pos,
            defining=0,
            implementing=0,
            module_name=self.module_name,
            base_type=None,
            objstruct_cname=self.objstruct_name,
            typeobj_cname=self.typeobj_name,
            visibility=self.visibility,
            typedef_flag=self.typedef_flag,
            api=self.api,
            buffer_defaults=self.buffer_defaults(env),
            shadow=self.shadow)
4544

William Stein's avatar
William Stein committed
4545 4546 4547 4548
    def analyse_declarations(self, env):
        #print "CClassDefNode.analyse_declarations:", self.class_name
        #print "...visibility =", self.visibility
        #print "...module_name =", self.module_name
4549

William Stein's avatar
William Stein committed
4550
        if env.in_cinclude and not self.objstruct_name:
4551
            error(self.pos, "Object struct name specification required for C class defined in 'extern from' block")
4552
        if self.decorators:
4553
            error(self.pos, "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
William Stein's avatar
William Stein committed
4554
        self.base_type = None
4555 4556
        # Now that module imports are cached, we need to
        # import the modules for extern classes.
4557 4558 4559 4560 4561 4562 4563
        if self.module_name:
            self.module = None
            for module in env.cimported_modules:
                if module.name == self.module_name:
                    self.module = module
            if self.module is None:
                self.module = ModuleScope(self.module_name, None, env.context)
4564
                self.module.has_extern_class = 1
Robert Bradshaw's avatar
Robert Bradshaw committed
4565
                env.add_imported_module(self.module)
4566

William Stein's avatar
William Stein committed
4567 4568
        if self.base_class_name:
            if self.base_class_module:
4569 4570 4571 4572
                base_class_scope = env.find_imported_module(self.base_class_module.split('.'), self.pos)
                if not base_class_scope:
                    error(self.pos, "'%s' is not a cimported module" % self.base_class_module)
                    return
William Stein's avatar
William Stein committed
4573 4574
            else:
                base_class_scope = env
4575 4576 4577 4578 4579 4580
            if self.base_class_name == 'object':
                # extension classes are special and don't need to inherit from object
                if base_class_scope is None or base_class_scope.lookup('object') is None:
                    self.base_class_name = None
                    self.base_class_module = None
                    base_class_scope = None
William Stein's avatar
William Stein committed
4581 4582 4583 4584 4585
            if base_class_scope:
                base_class_entry = base_class_scope.find(self.base_class_name, self.pos)
                if base_class_entry:
                    if not base_class_entry.is_type:
                        error(self.pos, "'%s' is not a type name" % self.base_class_name)
4586
                    elif not base_class_entry.type.is_extension_type and \
Stefan Behnel's avatar
Stefan Behnel committed
4587
                             not (base_class_entry.type.is_builtin_type and
4588
                                  base_class_entry.type.objstruct_cname):
William Stein's avatar
William Stein committed
4589 4590
                        error(self.pos, "'%s' is not an extension type" % self.base_class_name)
                    elif not base_class_entry.type.is_complete():
4591 4592 4593
                        error(self.pos, "Base class '%s' of type '%s' is incomplete" % (
                            self.base_class_name, self.class_name))
                    elif base_class_entry.type.scope and base_class_entry.type.scope.directives and \
4594
                             base_class_entry.type.is_final_type:
4595 4596
                        error(self.pos, "Base class '%s' of type '%s' is final" % (
                            self.base_class_name, self.class_name))
4597 4598 4599 4600
                    elif base_class_entry.type.is_builtin_type and \
                             base_class_entry.type.name in ('tuple', 'str', 'bytes'):
                        error(self.pos, "inheritance from PyVarObject types like '%s' is not currently supported"
                              % base_class_entry.type.name)
William Stein's avatar
William Stein committed
4601 4602
                    else:
                        self.base_type = base_class_entry.type
4603
                if env.directives.get('freelist', 0) > 0:
Stefan Behnel's avatar
Stefan Behnel committed
4604
                    warning(self.pos, "freelists cannot be used on subtypes, only the base class can manage them", 1)
4605

William Stein's avatar
William Stein committed
4606
        has_body = self.body is not None
4607 4608 4609 4610 4611 4612
        if has_body and self.base_type and not self.base_type.scope:
            # To properly initialize inherited attributes, the base type must
            # be analysed before this type.
            self.base_type.defered_declarations.append(lambda : self.analyse_declarations(env))
            return

4613
        if self.module_name and self.visibility != 'extern':
4614 4615 4616 4617 4618 4619
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return
        else:
            home_scope = env
4620 4621

        if self.visibility == 'extern':
4622
            if (self.module_name == '__builtin__' and
4623 4624
                    self.class_name in Builtin.builtin_types and
                    env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython
4625
                warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1)
4626

4627
        self.entry = home_scope.declare_c_class(
4628 4629 4630 4631 4632 4633 4634 4635 4636 4637 4638 4639 4640
            name=self.class_name,
            pos=self.pos,
            defining=has_body and self.in_pxd,
            implementing=has_body and not self.in_pxd,
            module_name=self.module_name,
            base_type=self.base_type,
            objstruct_cname=self.objstruct_name,
            typeobj_cname=self.typeobj_name,
            visibility=self.visibility,
            typedef_flag=self.typedef_flag,
            api=self.api,
            buffer_defaults=self.buffer_defaults(env),
            shadow=self.shadow)
4641

4642 4643
        if self.shadow:
            home_scope.lookup(self.class_name).as_variable = self.entry
4644
        if home_scope is not env and self.visibility == 'extern':
4645
            env.add_imported_entry(self.class_name, self.entry, self.pos)
4646
        self.scope = scope = self.entry.type.scope
4647 4648
        if scope is not None:
            scope.directives = env.directives
4649

4650
        if self.doc and Options.docstrings:
4651
            scope.doc = embed_position(self.pos, self.doc)
4652

William Stein's avatar
William Stein committed
4653 4654
        if has_body:
            self.body.analyse_declarations(scope)
4655
            dict_entry = self.scope.lookup_here("__dict__")
4656
            if dict_entry and dict_entry.is_variable and (not scope.defined and not scope.implemented):
4657 4658
                dict_entry.getter_cname = self.scope.mangle_internal("__dict__getter")
                self.scope.declare_property("__dict__", dict_entry.doc, dict_entry.pos)
William Stein's avatar
William Stein committed
4659 4660 4661 4662
            if self.in_pxd:
                scope.defined = 1
            else:
                scope.implemented = 1
4663

William Stein's avatar
William Stein committed
4664
        env.allocate_vtable_names(self.entry)
4665

4666 4667 4668
        for thunk in self.entry.type.defered_declarations:
            thunk()

William Stein's avatar
William Stein committed
4669 4670
    def analyse_expressions(self, env):
        if self.body:
Robert Bradshaw's avatar
Robert Bradshaw committed
4671
            scope = self.entry.type.scope
4672 4673
            self.body = self.body.analyse_expressions(scope)
        return self
4674

4675
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
4676
        if self.body:
4677 4678
            self.generate_lambda_definitions(self.scope, code)
            self.body.generate_function_definitions(self.scope, code)
4679

William Stein's avatar
William Stein committed
4680 4681 4682
    def generate_execution_code(self, code):
        # This is needed to generate evaluation code for
        # default values of method arguments.
4683
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4684 4685
        if self.body:
            self.body.generate_execution_code(code)
4686

4687 4688 4689
    def annotate(self, code):
        if self.body:
            self.body.annotate(code)
William Stein's avatar
William Stein committed
4690 4691 4692 4693 4694 4695


class PropertyNode(StatNode):
    #  Definition of a property in an extension type.
    #
    #  name   string
4696
    #  doc    EncodedString or None    Doc string
4697
    #  entry  Symtab.Entry
William Stein's avatar
William Stein committed
4698
    #  body   StatListNode
4699

4700 4701
    child_attrs = ["body"]

William Stein's avatar
William Stein committed
4702
    def analyse_declarations(self, env):
4703 4704 4705
        self.entry = env.declare_property(self.name, self.doc, self.pos)
        self.entry.scope.directives = env.directives
        self.body.analyse_declarations(self.entry.scope)
4706

William Stein's avatar
William Stein committed
4707
    def analyse_expressions(self, env):
4708 4709
        self.body = self.body.analyse_expressions(env)
        return self
4710

4711 4712
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
4713 4714 4715 4716

    def generate_execution_code(self, code):
        pass

4717 4718 4719
    def annotate(self, code):
        self.body.annotate(code)

William Stein's avatar
William Stein committed
4720 4721 4722 4723 4724

class GlobalNode(StatNode):
    # Global variable declaration.
    #
    # names    [string]
4725

4726 4727
    child_attrs = []

William Stein's avatar
William Stein committed
4728 4729 4730 4731 4732
    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_global(name, self.pos)

    def analyse_expressions(self, env):
4733
        return self
4734

William Stein's avatar
William Stein committed
4735 4736 4737 4738
    def generate_execution_code(self, code):
        pass


4739 4740 4741 4742 4743 4744 4745 4746 4747 4748 4749 4750
class NonlocalNode(StatNode):
    # Nonlocal variable declaration via the 'nonlocal' keyword.
    #
    # names    [string]

    child_attrs = []

    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_nonlocal(name, self.pos)

    def analyse_expressions(self, env):
4751
        return self
4752 4753 4754 4755 4756

    def generate_execution_code(self, code):
        pass


William Stein's avatar
William Stein committed
4757 4758 4759 4760
class ExprStatNode(StatNode):
    #  Expression used as a statement.
    #
    #  expr   ExprNode
4761 4762

    child_attrs = ["expr"]
4763

Robert Bradshaw's avatar
Robert Bradshaw committed
4764
    def analyse_declarations(self, env):
4765
        from . import ExprNodes
Robert Bradshaw's avatar
Robert Bradshaw committed
4766
        if isinstance(self.expr, ExprNodes.GeneralCallNode):
4767
            func = self.expr.function.as_cython_attribute()
Robert Bradshaw's avatar
Robert Bradshaw committed
4768 4769 4770 4771 4772 4773 4774 4775 4776
            if func == u'declare':
                args, kwds = self.expr.explicit_args_kwds()
                if len(args):
                    error(self.expr.pos, "Variable names must be specified.")
                for var, type_node in kwds.key_value_pairs:
                    type = type_node.analyse_as_type(env)
                    if type is None:
                        error(type_node.pos, "Unknown type")
                    else:
4777
                        env.declare_var(var.value, type, var.pos, is_cdef=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
4778
                self.__class__ = PassStatNode
4779

William Stein's avatar
William Stein committed
4780
    def analyse_expressions(self, env):
4781
        self.expr.result_is_used = False  # hint that .result() may safely be left empty
4782 4783
        self.expr = self.expr.analyse_expressions(env)
        return self
4784

4785
    def nogil_check(self, env):
4786
        if self.expr.type.is_pyobject and self.expr.is_temp:
4787 4788 4789 4790
            self.gil_error()

    gil_message = "Discarding owned Python object"

William Stein's avatar
William Stein committed
4791
    def generate_execution_code(self, code):
4792
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4793
        self.expr.generate_evaluation_code(code)
4794 4795
        if not self.expr.is_temp and self.expr.result():
            code.putln("%s;" % self.expr.result())
William Stein's avatar
William Stein committed
4796
        self.expr.generate_disposal_code(code)
4797
        self.expr.free_temps(code)
William Stein's avatar
William Stein committed
4798

4799 4800 4801
    def generate_function_definitions(self, env, code):
        self.expr.generate_function_definitions(env, code)

4802 4803 4804
    def annotate(self, code):
        self.expr.annotate(code)

William Stein's avatar
William Stein committed
4805 4806 4807 4808 4809 4810 4811 4812 4813 4814 4815

class AssignmentNode(StatNode):
    #  Abstract base class for assignment nodes.
    #
    #  The analyse_expressions and generate_execution_code
    #  phases of assignments are split into two sub-phases
    #  each, to enable all the right hand sides of a
    #  parallel assignment to be evaluated before assigning
    #  to any of the left hand sides.

    def analyse_expressions(self, env):
4816
        node = self.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
4817
        if isinstance(node, AssignmentNode) and not isinstance(node, ParallelAssignmentNode):
4818
            if node.rhs.type.is_ptr and node.rhs.is_ephemeral():
4819 4820
                error(self.pos, "Storing unsafe C derivative of temporary Python reference")
        return node
4821

4822 4823 4824
#       def analyse_expressions(self, env):
#           self.analyse_expressions_1(env)
#           self.analyse_expressions_2(env)
William Stein's avatar
William Stein committed
4825 4826

    def generate_execution_code(self, code):
4827
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
4828 4829
        self.generate_rhs_evaluation_code(code)
        self.generate_assignment_code(code)
4830

William Stein's avatar
William Stein committed
4831 4832 4833 4834 4835 4836

class SingleAssignmentNode(AssignmentNode):
    #  The simplest case:
    #
    #    a = b
    #
4837 4838 4839 4840
    #  lhs                      ExprNode      Left hand side
    #  rhs                      ExprNode      Right hand side
    #  first                    bool          Is this guaranteed the first assignment to lhs?
    #  is_overloaded_assignment bool          Is this assignment done via an overloaded operator=
4841 4842
    #  exception_check
    #  exception_value
4843

4844
    child_attrs = ["lhs", "rhs"]
4845
    first = False
4846
    is_overloaded_assignment = False
4847
    declaration_only = False
William Stein's avatar
William Stein committed
4848 4849

    def analyse_declarations(self, env):
4850
        from . import ExprNodes
4851

4852 4853
        # handle declarations of the form x = cython.foo()
        if isinstance(self.rhs, ExprNodes.CallNode):
4854
            func_name = self.rhs.function.as_cython_attribute()
4855 4856 4857
            if func_name:
                args, kwds = self.rhs.explicit_args_kwds()
                if func_name in ['declare', 'typedef']:
4858 4859
                    if len(args) > 2:
                        error(args[2].pos, "Invalid positional argument.")
4860
                        return
4861 4862 4863 4864 4865 4866 4867 4868
                    if kwds is not None:
                        kwdict = kwds.compile_time_value(None)
                        if func_name == 'typedef' or 'visibility' not in kwdict:
                            error(kwds.pos, "Invalid keyword argument.")
                            return
                        visibility = kwdict['visibility']
                    else:
                        visibility = 'private'
4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879 4880 4881 4882
                    type = args[0].analyse_as_type(env)
                    if type is None:
                        error(args[0].pos, "Unknown type")
                        return
                    lhs = self.lhs
                    if func_name == 'declare':
                        if isinstance(lhs, ExprNodes.NameNode):
                            vars = [(lhs.name, lhs.pos)]
                        elif isinstance(lhs, ExprNodes.TupleNode):
                            vars = [(var.name, var.pos) for var in lhs.args]
                        else:
                            error(lhs.pos, "Invalid declaration")
                            return
                        for var, pos in vars:
4883
                            env.declare_var(var, type, pos, is_cdef=True, visibility=visibility)
Robert Bradshaw's avatar
Robert Bradshaw committed
4884 4885 4886 4887 4888
                        if len(args) == 2:
                            # we have a value
                            self.rhs = args[1]
                        else:
                            self.declaration_only = True
4889
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
4890
                        self.declaration_only = True
4891 4892
                        if not isinstance(lhs, ExprNodes.NameNode):
                            error(lhs.pos, "Invalid declaration.")
4893
                        env.declare_typedef(lhs.name, type, self.pos, visibility='private')
4894

4895 4896 4897
                elif func_name in ['struct', 'union']:
                    self.declaration_only = True
                    if len(args) > 0 or kwds is None:
4898
                        error(self.rhs.pos, "Struct or union members must be given by name.")
4899 4900 4901 4902 4903 4904 4905 4906 4907 4908 4909 4910 4911 4912 4913 4914 4915
                        return
                    members = []
                    for member, type_node in kwds.key_value_pairs:
                        type = type_node.analyse_as_type(env)
                        if type is None:
                            error(type_node.pos, "Unknown type")
                        else:
                            members.append((member.value, type, member.pos))
                    if len(members) < len(kwds.key_value_pairs):
                        return
                    if not isinstance(self.lhs, ExprNodes.NameNode):
                        error(self.lhs.pos, "Invalid declaration.")
                    name = self.lhs.name
                    scope = StructOrUnionScope(name)
                    env.declare_struct_or_union(name, func_name, scope, False, self.rhs.pos)
                    for member, type, pos in members:
                        scope.declare_var(member, type, pos)
4916

Mark Florisson's avatar
Mark Florisson committed
4917 4918 4919 4920 4921 4922 4923 4924
                elif func_name == 'fused_type':
                    # dtype = cython.fused_type(...)
                    self.declaration_only = True
                    if kwds:
                        error(self.rhs.function.pos,
                              "fused_type does not take keyword arguments")

                    fusednode = FusedTypeNode(self.rhs.pos,
4925
                                              name=self.lhs.name, types=args)
Mark Florisson's avatar
Mark Florisson committed
4926 4927
                    fusednode.analyse_declarations(env)

4928 4929 4930 4931
        if self.declaration_only:
            return
        else:
            self.lhs.analyse_target_declaration(env)
4932

4933 4934
    def analyse_types(self, env, use_temp=0):
        from . import ExprNodes
4935

4936
        self.rhs = self.rhs.analyse_types(env)
4937

4938 4939 4940
        unrolled_assignment = self.unroll_rhs(env)
        if unrolled_assignment:
            return unrolled_assignment
Robert Bradshaw's avatar
Robert Bradshaw committed
4941

4942
        self.lhs = self.lhs.analyse_target_types(env)
4943
        self.lhs.gil_assignment_check(env)
4944 4945 4946
        unrolled_assignment = self.unroll_lhs(env)
        if unrolled_assignment:
            return unrolled_assignment
4947

4948 4949 4950
        if isinstance(self.lhs, ExprNodes.MemoryViewIndexNode):
            self.lhs.analyse_broadcast_operation(self.rhs)
            self.lhs = self.lhs.analyse_as_memview_scalar_assignment(self.rhs)
4951
        elif self.lhs.type.is_array:
4952
            if not isinstance(self.lhs, ExprNodes.SliceIndexNode):
4953
                # cannot assign to C array, only to its full slice
4954
                self.lhs = ExprNodes.SliceIndexNode(self.lhs.pos, base=self.lhs, start=None, stop=None)
4955
                self.lhs = self.lhs.analyse_target_types(env)
4956

4957 4958 4959 4960
        if self.lhs.type.is_cpp_class:
            op = env.lookup_operator_for_types(self.pos, '=', [self.lhs.type, self.rhs.type])
            if op:
                rhs = self.rhs
4961
                self.is_overloaded_assignment = True
4962 4963 4964 4965
                self.exception_check = op.type.exception_check
                self.exception_value = op.type.exception_value
                if self.exception_check == '+' and self.exception_value is None:
                    env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp"))
4966
            else:
4967
                rhs = self.rhs.coerce_to(self.lhs.type, env)
4968
        else:
4969 4970
            rhs = self.rhs.coerce_to(self.lhs.type, env)

4971 4972 4973
        if use_temp or rhs.is_attribute or (
                not rhs.is_name and not rhs.is_literal and
                rhs.type.is_pyobject):
4974
            # things like (cdef) attribute access are not safe (traverses pointers)
4975 4976 4977 4978
            rhs = rhs.coerce_to_temp(env)
        elif rhs.type.is_pyobject:
            rhs = rhs.coerce_to_simple(env)
        self.rhs = rhs
4979
        return self
4980

4981 4982
    def unroll(self, node, target_size, env):
        from . import ExprNodes, UtilNodes
4983 4984 4985 4986

        base = node
        start_node = stop_node = step_node = check_node = None

4987
        if node.type.is_ctuple:
4988
            slice_size = node.type.size
4989

4990 4991 4992
        elif node.type.is_ptr or node.type.is_array:
            while isinstance(node, ExprNodes.SliceIndexNode) and not (node.start or node.stop):
                base = node = node.base
4993 4994 4995
            if isinstance(node, ExprNodes.SliceIndexNode):
                base = node.base
                start_node = node.start
4996 4997
                if start_node:
                    start_node = start_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
4998
                stop_node = node.stop
4999 5000 5001
                if stop_node:
                    stop_node = stop_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
                else:
5002
                    if node.type.is_array and node.type.size:
5003 5004
                        stop_node = ExprNodes.IntNode(
                            self.pos, value=str(node.type.size),
5005
                            constant_result=(node.type.size if isinstance(node.type.size, _py_int_types)
5006
                                             else ExprNodes.constant_value_not_set))
5007 5008 5009
                    else:
                        error(self.pos, "C array iteration requires known end index")
                        return
5010
                step_node = None  #node.step
5011 5012
                if step_node:
                    step_node = step_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5013

5014 5015 5016 5017
                # TODO: Factor out SliceIndexNode.generate_slice_guard_code() for use here.
                def get_const(node, none_value):
                    if node is None:
                        return none_value
5018
                    elif node.has_constant_result():
5019 5020
                        return node.constant_result
                    else:
5021 5022
                        raise ValueError("Not a constant.")

5023 5024 5025 5026 5027
                try:
                    slice_size = (get_const(stop_node, None) - get_const(start_node, 0)) / get_const(step_node, 1)
                except ValueError:
                    error(self.pos, "C array assignment currently requires known endpoints")
                    return
5028 5029 5030

            elif node.type.is_array:
                slice_size = node.type.size
5031
                if not isinstance(slice_size, _py_int_types):
5032
                    return  # might still work when coercing to Python
5033 5034 5035 5036 5037 5038
            else:
                return

        else:
            return

5039 5040 5041 5042 5043
        if slice_size != target_size:
            error(self.pos, "Assignment to/from slice of wrong length, expected %s, got %s" % (
                slice_size, target_size))
            return

5044
        items = []
5045 5046 5047
        base = UtilNodes.LetRefNode(base)
        refs = [base]
        if start_node and not start_node.is_literal:
5048 5049
            start_node = UtilNodes.LetRefNode(start_node)
            refs.append(start_node)
5050
        if stop_node and not stop_node.is_literal:
5051 5052
            stop_node = UtilNodes.LetRefNode(stop_node)
            refs.append(stop_node)
5053
        if step_node and not step_node.is_literal:
5054 5055
            step_node = UtilNodes.LetRefNode(step_node)
            refs.append(step_node)
5056

5057
        for ix in range(target_size):
5058
            ix_node = ExprNodes.IntNode(self.pos, value=str(ix), constant_result=ix, type=PyrexTypes.c_py_ssize_t_type)
5059
            if step_node is not None:
5060 5061 5062 5063 5064
                if step_node.has_constant_result():
                    step_value = ix_node.constant_result * step_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(step_value), constant_result=step_value)
                else:
                    ix_node = ExprNodes.MulNode(self.pos, operator='*', operand1=step_node, operand2=ix_node)
5065
            if start_node is not None:
5066 5067 5068 5069 5070 5071 5072
                if start_node.has_constant_result() and ix_node.has_constant_result():
                    index_value = ix_node.constant_result + start_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(index_value), constant_result=index_value)
                else:
                    ix_node = ExprNodes.AddNode(
                        self.pos, operator='+', operand1=start_node, operand2=ix_node)
            items.append(ExprNodes.IndexNode(self.pos, base=base, index=ix_node.analyse_types(env)))
5073 5074
        return check_node, refs, items

5075
    def unroll_assignments(self, refs, check_node, lhs_list, rhs_list, env):
5076
        from . import UtilNodes
5077
        assignments = []
5078
        for lhs, rhs in zip(lhs_list, rhs_list):
5079
            assignments.append(SingleAssignmentNode(self.pos, lhs=lhs, rhs=rhs, first=self.first))
5080
        node = ParallelAssignmentNode(pos=self.pos, stats=assignments).analyse_expressions(env)
5081
        if check_node:
5082
            node = StatListNode(pos=self.pos, stats=[check_node, node])
5083
        for ref in refs[::-1]:
5084 5085
            node = UtilNodes.LetNode(ref, node)
        return node
5086

5087
    def unroll_rhs(self, env):
5088
        from . import ExprNodes
5089 5090
        if not isinstance(self.lhs, ExprNodes.TupleNode):
            return
5091 5092
        if any(arg.is_starred for arg in self.lhs.args):
            return
5093 5094 5095 5096 5097 5098 5099 5100 5101 5102 5103

        unrolled = self.unroll(self.rhs, len(self.lhs.args), env)
        if not unrolled:
            return
        check_node, refs, rhs = unrolled
        return self.unroll_assignments(refs, check_node, self.lhs.args, rhs, env)

    def unroll_lhs(self, env):
        if self.lhs.type.is_ctuple:
            # Handled directly.
            return
5104
        from . import ExprNodes
5105 5106 5107 5108 5109 5110 5111 5112 5113
        if not isinstance(self.rhs, ExprNodes.TupleNode):
            return

        unrolled = self.unroll(self.lhs, len(self.rhs.args), env)
        if not unrolled:
            return
        check_node, refs, lhs = unrolled
        return self.unroll_assignments(refs, check_node, lhs, self.rhs.args, env)

William Stein's avatar
William Stein committed
5114 5115
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5116

5117
    def generate_assignment_code(self, code, overloaded_assignment=False):
5118 5119 5120 5121 5122 5123 5124 5125 5126
        if self.is_overloaded_assignment:
            self.lhs.generate_assignment_code(
                self.rhs,
                code,
                overloaded_assignment=self.is_overloaded_assignment,
                exception_check=self.exception_check,
                exception_value=self.exception_value)
        else:
            self.lhs.generate_assignment_code(self.rhs, code)
William Stein's avatar
William Stein committed
5127

5128 5129 5130
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5131 5132 5133 5134
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

William Stein's avatar
William Stein committed
5135 5136 5137 5138 5139 5140 5141 5142 5143 5144 5145

class CascadedAssignmentNode(AssignmentNode):
    #  An assignment with multiple left hand sides:
    #
    #    a = b = c
    #
    #  lhs_list   [ExprNode]   Left hand sides
    #  rhs        ExprNode     Right hand sides
    #
    #  Used internally:
    #
5146 5147 5148
    #  coerced_values       [ExprNode]   RHS coerced to all distinct LHS types
    #  cloned_values        [ExprNode]   cloned RHS value for each LHS
    #  assignment_overloads [Bool]       If each assignment uses a C++ operator=
5149

5150 5151
    child_attrs = ["lhs_list", "rhs", "coerced_values", "cloned_values"]
    cloned_values = None
5152
    coerced_values = None
5153
    assignment_overloads = None
5154

William Stein's avatar
William Stein committed
5155 5156 5157
    def analyse_declarations(self, env):
        for lhs in self.lhs_list:
            lhs.analyse_target_declaration(env)
5158

5159
    def analyse_types(self, env, use_temp=0):
5160
        from .ExprNodes import CloneNode, ProxyNode
5161

5162
        # collect distinct types used on the LHS
5163
        lhs_types = set()
5164 5165
        for i, lhs in enumerate(self.lhs_list):
            lhs = self.lhs_list[i] = lhs.analyse_target_types(env)
5166 5167 5168
            lhs.gil_assignment_check(env)
            lhs_types.add(lhs.type)

5169
        rhs = self.rhs.analyse_types(env)
5170
        # common special case: only one type needed on the LHS => coerce only once
5171
        if len(lhs_types) == 1:
5172 5173 5174 5175 5176 5177 5178
            # Avoid coercion for overloaded assignment operators.
            if next(iter(lhs_types)).is_cpp_class:
                op = env.lookup_operator('=', [lhs, self.rhs])
                if not op:
                    rhs = rhs.coerce_to(lhs_types.pop(), env)
            else:
                rhs = rhs.coerce_to(lhs_types.pop(), env)
5179 5180 5181

        if not rhs.is_name and not rhs.is_literal and (
                use_temp or rhs.is_attribute or rhs.type.is_pyobject):
5182
            rhs = rhs.coerce_to_temp(env)
5183
        else:
5184
            rhs = rhs.coerce_to_simple(env)
5185 5186
        self.rhs = ProxyNode(rhs) if rhs.is_temp else rhs

5187
        # clone RHS and coerce it to all distinct LHS types
5188 5189
        self.coerced_values = []
        coerced_values = {}
5190
        self.assignment_overloads = []
5191
        for lhs in self.lhs_list:
5192 5193
            overloaded = lhs.type.is_cpp_class and env.lookup_operator('=', [lhs, self.rhs])
            self.assignment_overloads.append(overloaded)
5194
            if lhs.type not in coerced_values and lhs.type != rhs.type:
5195
                rhs = CloneNode(self.rhs)
5196
                if not overloaded:
5197
                    rhs = rhs.coerce_to(lhs.type, env)
5198 5199
                self.coerced_values.append(rhs)
                coerced_values[lhs.type] = rhs
5200

5201 5202
        # clone coerced values for all LHS assignments
        self.cloned_values = []
William Stein's avatar
William Stein committed
5203
        for lhs in self.lhs_list:
5204
            rhs = coerced_values.get(lhs.type, self.rhs)
5205
            self.cloned_values.append(CloneNode(rhs))
5206
        return self
5207

William Stein's avatar
William Stein committed
5208 5209
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5210

5211
    def generate_assignment_code(self, code, overloaded_assignment=False):
5212
        # prepare all coercions
5213 5214
        for rhs in self.coerced_values:
            rhs.generate_evaluation_code(code)
5215
        # assign clones to LHS
5216
        for lhs, rhs, overload in zip(self.lhs_list, self.cloned_values, self.assignment_overloads):
William Stein's avatar
William Stein committed
5217
            rhs.generate_evaluation_code(code)
5218
            lhs.generate_assignment_code(rhs, code, overloaded_assignment=overload)
5219 5220 5221 5222
        # dispose of coerced values and original RHS
        for rhs_value in self.coerced_values:
            rhs_value.generate_disposal_code(code)
            rhs_value.free_temps(code)
William Stein's avatar
William Stein committed
5223
        self.rhs.generate_disposal_code(code)
5224
        self.rhs.free_temps(code)
William Stein's avatar
William Stein committed
5225

5226 5227 5228
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5229
    def annotate(self, code):
5230 5231
        for rhs in self.coerced_values:
            rhs.annotate(code)
5232
        for lhs, rhs in zip(self.lhs_list, self.cloned_values):
5233 5234
            lhs.annotate(code)
            rhs.annotate(code)
5235
        self.rhs.annotate(code)
5236

5237

William Stein's avatar
William Stein committed
5238 5239 5240 5241 5242 5243 5244 5245 5246 5247 5248 5249 5250
class ParallelAssignmentNode(AssignmentNode):
    #  A combined packing/unpacking assignment:
    #
    #    a, b, c =  d, e, f
    #
    #  This has been rearranged by the parser into
    #
    #    a = d ; b = e ; c = f
    #
    #  but we must evaluate all the right hand sides
    #  before assigning to any of the left hand sides.
    #
    #  stats     [AssignmentNode]   The constituent assignments
5251

5252 5253
    child_attrs = ["stats"]

William Stein's avatar
William Stein committed
5254 5255 5256
    def analyse_declarations(self, env):
        for stat in self.stats:
            stat.analyse_declarations(env)
5257

William Stein's avatar
William Stein committed
5258
    def analyse_expressions(self, env):
5259 5260
        self.stats = [stat.analyse_types(env, use_temp=1)
                      for stat in self.stats]
5261
        return self
5262

Robert Bradshaw's avatar
Robert Bradshaw committed
5263 5264
#    def analyse_expressions(self, env):
#        for stat in self.stats:
5265
#            stat.analyse_expressions_1(env, use_temp=1)
Robert Bradshaw's avatar
Robert Bradshaw committed
5266 5267
#        for stat in self.stats:
#            stat.analyse_expressions_2(env)
5268

William Stein's avatar
William Stein committed
5269
    def generate_execution_code(self, code):
5270
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5271 5272 5273 5274 5275
        for stat in self.stats:
            stat.generate_rhs_evaluation_code(code)
        for stat in self.stats:
            stat.generate_assignment_code(code)

5276 5277 5278 5279
    def generate_function_definitions(self, env, code):
        for stat in self.stats:
            stat.generate_function_definitions(env, code)

5280 5281 5282 5283 5284
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)


5285
class InPlaceAssignmentNode(AssignmentNode):
Craig Citro's avatar
Craig Citro committed
5286
    #  An in place arithmetic operand:
5287 5288 5289 5290 5291 5292 5293
    #
    #    a += b
    #    a -= b
    #    ...
    #
    #  lhs      ExprNode      Left hand side
    #  rhs      ExprNode      Right hand side
Stefan Behnel's avatar
Stefan Behnel committed
5294
    #  operator char          one of "+-*/%^&|"
5295
    #
5296 5297 5298 5299 5300 5301 5302
    #  This code is a bit tricky because in order to obey Python
    #  semantics the sub-expressions (e.g. indices) of the lhs must
    #  not be evaluated twice. So we must re-use the values calculated
    #  in evaluation phase for the assignment phase as well.
    #  Fortunately, the type of the lhs node is fairly constrained
    #  (it must be a NameNode, AttributeNode, or IndexNode).

5303
    child_attrs = ["lhs", "rhs"]
5304

5305 5306
    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)
5307

5308
    def analyse_types(self, env):
5309 5310
        self.rhs = self.rhs.analyse_types(env)
        self.lhs = self.lhs.analyse_target_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5311

5312
        # When assigning to a fully indexed buffer or memoryview, coerce the rhs
5313
        if self.lhs.is_memview_index or self.lhs.is_buffer_access:
5314
            self.rhs = self.rhs.coerce_to(self.lhs.type, env)
5315 5316 5317
        elif self.lhs.type.is_string and self.operator in '+-':
            # use pointer arithmetic for char* LHS instead of string concat
            self.rhs = self.rhs.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5318
        return self
5319

5320
    def generate_execution_code(self, code):
5321
        code.mark_pos(self.pos)
5322 5323 5324
        lhs, rhs = self.lhs, self.rhs
        rhs.generate_evaluation_code(code)
        lhs.generate_subexpr_evaluation_code(code)
5325 5326 5327 5328 5329
        c_op = self.operator
        if c_op == "//":
            c_op = "/"
        elif c_op == "**":
            error(self.pos, "No C inplace power operator")
5330 5331
        if lhs.is_buffer_access or lhs.is_memview_index:
            if lhs.type.is_pyobject:
5332
                error(self.pos, "In-place operators not allowed on object buffers in this release.")
5333
            if c_op in ('/', '%') and lhs.type.is_int and not code.globalstate.directives['cdivision']:
Robert Bradshaw's avatar
Robert Bradshaw committed
5334
                error(self.pos, "In-place non-c divide operators not allowed on int buffers.")
5335 5336 5337
            lhs.generate_buffer_setitem_code(rhs, code, c_op)
        elif lhs.is_memview_slice:
            error(self.pos, "Inplace operators not supported on memoryview slices")
5338 5339 5340
        else:
            # C++
            # TODO: make sure overload is declared
5341 5342 5343 5344 5345
            code.putln("%s %s= %s;" % (lhs.result(), c_op, rhs.result()))
        lhs.generate_subexpr_disposal_code(code)
        lhs.free_subexpr_temps(code)
        rhs.generate_disposal_code(code)
        rhs.free_temps(code)
5346

5347 5348 5349
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)
5350

5351
    def create_binop_node(self):
5352
        from . import ExprNodes
5353
        return ExprNodes.binop_node(self.pos, self.operator, self.lhs, self.rhs)
5354

William Stein's avatar
William Stein committed
5355 5356 5357 5358

class PrintStatNode(StatNode):
    #  print statement
    #
5359
    #  arg_tuple         TupleNode
5360
    #  stream            ExprNode or None (stdout)
5361
    #  append_newline    boolean
5362

5363
    child_attrs = ["arg_tuple", "stream"]
5364

William Stein's avatar
William Stein committed
5365
    def analyse_expressions(self, env):
5366
        if self.stream:
5367 5368 5369 5370
            stream = self.stream.analyse_expressions(env)
            self.stream = stream.coerce_to_pyobject(env)
        arg_tuple = self.arg_tuple.analyse_expressions(env)
        self.arg_tuple = arg_tuple.coerce_to_pyobject(env)
5371
        env.use_utility_code(printing_utility_code)
5372 5373
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            env.use_utility_code(printing_one_utility_code)
5374
        return self
5375

5376
    nogil_check = Node.gil_error
5377
    gil_message = "Python print statement"
5378

William Stein's avatar
William Stein committed
5379
    def generate_execution_code(self, code):
5380
        code.mark_pos(self.pos)
5381 5382 5383 5384 5385
        if self.stream:
            self.stream.generate_evaluation_code(code)
            stream_result = self.stream.py_result()
        else:
            stream_result = '0'
5386 5387 5388
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            arg = self.arg_tuple.args[0]
            arg.generate_evaluation_code(code)
5389

5390
            code.putln(
5391 5392
                "if (__Pyx_PrintOne(%s, %s) < 0) %s" % (
                    stream_result,
5393 5394 5395 5396 5397 5398 5399
                    arg.py_result(),
                    code.error_goto(self.pos)))
            arg.generate_disposal_code(code)
            arg.free_temps(code)
        else:
            self.arg_tuple.generate_evaluation_code(code)
            code.putln(
5400 5401
                "if (__Pyx_Print(%s, %s, %d) < 0) %s" % (
                    stream_result,
5402 5403 5404 5405 5406
                    self.arg_tuple.py_result(),
                    self.append_newline,
                    code.error_goto(self.pos)))
            self.arg_tuple.generate_disposal_code(code)
            self.arg_tuple.free_temps(code)
5407

5408 5409 5410 5411
        if self.stream:
            self.stream.generate_disposal_code(code)
            self.stream.free_temps(code)

5412
    def generate_function_definitions(self, env, code):
5413 5414
        if self.stream:
            self.stream.generate_function_definitions(env, code)
5415
        self.arg_tuple.generate_function_definitions(env, code)
5416

5417
    def annotate(self, code):
5418 5419
        if self.stream:
            self.stream.annotate(code)
5420
        self.arg_tuple.annotate(code)
William Stein's avatar
William Stein committed
5421 5422


5423 5424 5425 5426 5427 5428 5429 5430 5431
class ExecStatNode(StatNode):
    #  exec statement
    #
    #  args     [ExprNode]

    child_attrs = ["args"]

    def analyse_expressions(self, env):
        for i, arg in enumerate(self.args):
5432
            arg = arg.analyse_expressions(env)
5433 5434 5435
            arg = arg.coerce_to_pyobject(env)
            self.args[i] = arg
        env.use_utility_code(Builtin.pyexec_utility_code)
5436
        return self
5437

5438
    nogil_check = Node.gil_error
5439 5440 5441
    gil_message = "Python exec statement"

    def generate_execution_code(self, code):
5442
        code.mark_pos(self.pos)
5443 5444 5445
        args = []
        for arg in self.args:
            arg.generate_evaluation_code(code)
5446
            args.append(arg.py_result())
5447
        args = tuple(args + ['0', '0'][:3-len(args)])
5448
        temp_result = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
5449
        code.putln("%s = __Pyx_PyExec3(%s, %s, %s);" % ((temp_result,) + args))
5450 5451
        for arg in self.args:
            arg.generate_disposal_code(code)
5452
            arg.free_temps(code)
5453
        code.putln(
5454 5455 5456 5457
            code.error_goto_if_null(temp_result, self.pos))
        code.put_gotref(temp_result)
        code.put_decref_clear(temp_result, py_object_type)
        code.funcstate.release_temp(temp_result)
5458 5459 5460 5461 5462 5463

    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)


William Stein's avatar
William Stein committed
5464 5465 5466 5467
class DelStatNode(StatNode):
    #  del statement
    #
    #  args     [ExprNode]
5468

5469
    child_attrs = ["args"]
5470
    ignore_nonexisting = False
5471

William Stein's avatar
William Stein committed
5472 5473 5474
    def analyse_declarations(self, env):
        for arg in self.args:
            arg.analyse_target_declaration(env)
5475

William Stein's avatar
William Stein committed
5476
    def analyse_expressions(self, env):
5477 5478
        for i, arg in enumerate(self.args):
            arg = self.args[i] = arg.analyse_target_expression(env, None)
5479
            if arg.type.is_pyobject or (arg.is_name and arg.type.is_memoryviewslice):
5480 5481
                if arg.is_name and arg.entry.is_cglobal:
                    error(arg.pos, "Deletion of global C variable")
Robert Bradshaw's avatar
Robert Bradshaw committed
5482
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
Robert Bradshaw's avatar
Robert Bradshaw committed
5483
                self.cpp_check(env)
5484
            elif arg.type.is_cpp_class:
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5485
                error(arg.pos, "Deletion of non-heap C++ object")
5486 5487
            elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
                pass  # del ba[i]
5488
            else:
Robert Bradshaw's avatar
Robert Bradshaw committed
5489
                error(arg.pos, "Deletion of non-Python, non-C++ object")
5490
            #arg.release_target_temp(env)
5491
        return self
5492

5493
    def nogil_check(self, env):
5494 5495
        for arg in self.args:
            if arg.type.is_pyobject:
5496
                self.gil_error()
5497

5498 5499
    gil_message = "Deleting Python object"

William Stein's avatar
William Stein committed
5500
    def generate_execution_code(self, code):
5501
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5502
        for arg in self.args:
5503 5504 5505
            if (arg.type.is_pyobject or
                    arg.type.is_memoryviewslice or
                    arg.is_subscript and arg.base.type is Builtin.bytearray_type):
5506 5507
                arg.generate_deletion_code(
                    code, ignore_nonexisting=self.ignore_nonexisting)
5508
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
5509
                arg.generate_evaluation_code(code)
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5510
                code.putln("delete %s;" % arg.result())
5511
                arg.generate_disposal_code(code)
William Stein's avatar
William Stein committed
5512 5513
            # else error reported earlier

5514 5515 5516 5517
    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)

William Stein's avatar
William Stein committed
5518 5519 5520

class PassStatNode(StatNode):
    #  pass statement
5521 5522

    child_attrs = []
5523

William Stein's avatar
William Stein committed
5524
    def analyse_expressions(self, env):
5525
        return self
5526

William Stein's avatar
William Stein committed
5527 5528 5529 5530
    def generate_execution_code(self, code):
        pass


5531 5532 5533 5534 5535 5536 5537 5538 5539
class IndirectionNode(StatListNode):
    """
    This adds an indirection so that the node can be shared and a subtree can
    be removed at any time by clearing self.stats.
    """

    def __init__(self, stats):
        super(IndirectionNode, self).__init__(stats[0].pos, stats=stats)

5540

William Stein's avatar
William Stein committed
5541 5542
class BreakStatNode(StatNode):

5543
    child_attrs = []
5544
    is_terminator = True
5545

William Stein's avatar
William Stein committed
5546
    def analyse_expressions(self, env):
5547
        return self
5548

William Stein's avatar
William Stein committed
5549
    def generate_execution_code(self, code):
5550
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5551 5552 5553
        if not code.break_label:
            error(self.pos, "break statement not inside loop")
        else:
5554
            code.put_goto(code.break_label)
William Stein's avatar
William Stein committed
5555 5556 5557 5558


class ContinueStatNode(StatNode):

5559
    child_attrs = []
5560
    is_terminator = True
5561

William Stein's avatar
William Stein committed
5562
    def analyse_expressions(self, env):
5563
        return self
5564

William Stein's avatar
William Stein committed
5565
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
5566
        if not code.continue_label:
William Stein's avatar
William Stein committed
5567
            error(self.pos, "continue statement not inside loop")
Stefan Behnel's avatar
Stefan Behnel committed
5568 5569 5570
            return
        code.mark_pos(self.pos)
        code.put_goto(code.continue_label)
William Stein's avatar
William Stein committed
5571 5572 5573 5574 5575 5576 5577


class ReturnStatNode(StatNode):
    #  return statement
    #
    #  value         ExprNode or None
    #  return_type   PyrexType
5578
    #  in_generator  return inside of generator => raise StopIteration
5579

5580
    child_attrs = ["value"]
5581
    is_terminator = True
5582
    in_generator = False
5583

5584 5585 5586
    # Whether we are in a parallel section
    in_parallel = False

William Stein's avatar
William Stein committed
5587 5588 5589 5590 5591
    def analyse_expressions(self, env):
        return_type = env.return_type
        self.return_type = return_type
        if not return_type:
            error(self.pos, "Return not inside a function body")
5592
            return self
William Stein's avatar
William Stein committed
5593
        if self.value:
5594
            self.value = self.value.analyse_types(env)
William Stein's avatar
William Stein committed
5595
            if return_type.is_void or return_type.is_returncode:
5596
                error(self.value.pos, "Return with value in void function")
William Stein's avatar
William Stein committed
5597 5598 5599 5600
            else:
                self.value = self.value.coerce_to(env.return_type, env)
        else:
            if (not return_type.is_void
5601 5602 5603
                    and not return_type.is_pyobject
                    and not return_type.is_returncode):
                error(self.pos, "Return value required")
5604
        return self
5605

5606
    def nogil_check(self, env):
5607
        if self.return_type.is_pyobject:
5608
            self.gil_error()
5609 5610 5611

    gil_message = "Returning Python object"

William Stein's avatar
William Stein committed
5612
    def generate_execution_code(self, code):
5613
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5614 5615 5616
        if not self.return_type:
            # error reported earlier
            return
5617 5618 5619
        if self.return_type.is_pyobject:
            code.put_xdecref(Naming.retval_cname,
                             self.return_type)
5620

William Stein's avatar
William Stein committed
5621 5622
        if self.value:
            self.value.generate_evaluation_code(code)
5623
            if self.return_type.is_memoryviewslice:
5624
                from . import MemoryView
5625
                MemoryView.put_acquire_memoryviewslice(
5626 5627 5628 5629 5630 5631
                    lhs_cname=Naming.retval_cname,
                    lhs_type=self.return_type,
                    lhs_pos=self.value.pos,
                    rhs=self.value,
                    code=code,
                    have_gil=self.in_nogil_context)
5632 5633
            elif self.in_generator:
                # return value == raise StopIteration(value), but uncatchable
5634
                code.globalstate.use_utility_code(
5635
                    UtilityCode.load_cached("ReturnWithStopIteration", "Coroutine.c"))
5636 5637 5638
                code.putln("%s = NULL; __Pyx_ReturnWithStopIteration(%s);" % (
                    Naming.retval_cname,
                    self.value.py_result()))
5639
                self.value.generate_disposal_code(code)
5640 5641
            else:
                self.value.make_owned_reference(code)
5642 5643 5644
                code.putln("%s = %s;" % (
                    Naming.retval_cname,
                    self.value.result_as(self.return_type)))
5645
            self.value.generate_post_assignment_code(code)
5646
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
5647 5648
        else:
            if self.return_type.is_pyobject:
5649 5650 5651 5652
                if self.in_generator:
                    code.putln("%s = NULL;" % Naming.retval_cname)
                else:
                    code.put_init_to_py_none(Naming.retval_cname, self.return_type)
William Stein's avatar
William Stein committed
5653
            elif self.return_type.is_returncode:
5654 5655
                self.put_return(code, self.return_type.default_value)

5656
        for cname, type in code.funcstate.temps_holding_reference():
5657
            code.put_decref_clear(cname, type)
5658

5659
        code.put_goto(code.return_label)
5660

5661 5662 5663 5664 5665
    def put_return(self, code, value):
        if self.in_parallel:
            code.putln_openmp("#pragma omp critical(__pyx_returning)")
        code.putln("%s = %s;" % (Naming.retval_cname, value))

5666 5667 5668
    def generate_function_definitions(self, env, code):
        if self.value is not None:
            self.value.generate_function_definitions(env, code)
5669

5670 5671 5672
    def annotate(self, code):
        if self.value:
            self.value.annotate(code)
William Stein's avatar
William Stein committed
5673 5674 5675 5676 5677 5678 5679 5680


class RaiseStatNode(StatNode):
    #  raise statement
    #
    #  exc_type    ExprNode or None
    #  exc_value   ExprNode or None
    #  exc_tb      ExprNode or None
Haoyu Bai's avatar
Haoyu Bai committed
5681
    #  cause       ExprNode or None
5682

Haoyu Bai's avatar
Haoyu Bai committed
5683
    child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"]
5684
    is_terminator = True
5685

William Stein's avatar
William Stein committed
5686 5687
    def analyse_expressions(self, env):
        if self.exc_type:
5688 5689
            exc_type = self.exc_type.analyse_types(env)
            self.exc_type = exc_type.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
5690
        if self.exc_value:
5691 5692
            exc_value = self.exc_value.analyse_types(env)
            self.exc_value = exc_value.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
5693
        if self.exc_tb:
5694 5695
            exc_tb = self.exc_tb.analyse_types(env)
            self.exc_tb = exc_tb.coerce_to_pyobject(env)
Haoyu Bai's avatar
Haoyu Bai committed
5696
        if self.cause:
5697 5698
            cause = self.cause.analyse_types(env)
            self.cause = cause.coerce_to_pyobject(env)
5699 5700 5701 5702
        # special cases for builtin exceptions
        self.builtin_exc_name = None
        if self.exc_type and not self.exc_value and not self.exc_tb:
            exc = self.exc_type
5703
            from . import ExprNodes
Robert Bradshaw's avatar
Robert Bradshaw committed
5704
            if (isinstance(exc, ExprNodes.SimpleCallNode) and
5705 5706
                    not (exc.args or (exc.arg_tuple is not None and exc.arg_tuple.args))):
                exc = exc.function  # extract the exception type
5707 5708 5709 5710
            if exc.is_name and exc.entry.is_builtin:
                self.builtin_exc_name = exc.name
                if self.builtin_exc_name == 'MemoryError':
                    self.exc_type = None # has a separate implementation
5711
        return self
5712

5713
    nogil_check = Node.gil_error
5714 5715
    gil_message = "Raising exception"

William Stein's avatar
William Stein committed
5716
    def generate_execution_code(self, code):
5717
        code.mark_pos(self.pos)
5718 5719 5720 5721
        if self.builtin_exc_name == 'MemoryError':
            code.putln('PyErr_NoMemory(); %s' % code.error_goto(self.pos))
            return

William Stein's avatar
William Stein committed
5722 5723 5724
        if self.exc_type:
            self.exc_type.generate_evaluation_code(code)
            type_code = self.exc_type.py_result()
5725 5726
            if self.exc_type.is_name:
                code.globalstate.use_entry_utility_code(self.exc_type.entry)
William Stein's avatar
William Stein committed
5727
        else:
Stefan Behnel's avatar
Stefan Behnel committed
5728
            type_code = "0"
William Stein's avatar
William Stein committed
5729 5730 5731 5732 5733 5734 5735 5736 5737 5738
        if self.exc_value:
            self.exc_value.generate_evaluation_code(code)
            value_code = self.exc_value.py_result()
        else:
            value_code = "0"
        if self.exc_tb:
            self.exc_tb.generate_evaluation_code(code)
            tb_code = self.exc_tb.py_result()
        else:
            tb_code = "0"
Haoyu Bai's avatar
Haoyu Bai committed
5739 5740 5741 5742 5743
        if self.cause:
            self.cause.generate_evaluation_code(code)
            cause_code = self.cause.py_result()
        else:
            cause_code = "0"
5744
        code.globalstate.use_utility_code(raise_utility_code)
5745
        code.putln(
Haoyu Bai's avatar
Haoyu Bai committed
5746
            "__Pyx_Raise(%s, %s, %s, %s);" % (
5747 5748
                type_code,
                value_code,
Haoyu Bai's avatar
Haoyu Bai committed
5749 5750 5751
                tb_code,
                cause_code))
        for obj in (self.exc_type, self.exc_value, self.exc_tb, self.cause):
5752 5753 5754
            if obj:
                obj.generate_disposal_code(code)
                obj.free_temps(code)
William Stein's avatar
William Stein committed
5755 5756 5757
        code.putln(
            code.error_goto(self.pos))

5758 5759 5760 5761 5762 5763 5764
    def generate_function_definitions(self, env, code):
        if self.exc_type is not None:
            self.exc_type.generate_function_definitions(env, code)
        if self.exc_value is not None:
            self.exc_value.generate_function_definitions(env, code)
        if self.exc_tb is not None:
            self.exc_tb.generate_function_definitions(env, code)
Haoyu Bai's avatar
Haoyu Bai committed
5765 5766
        if self.cause is not None:
            self.cause.generate_function_definitions(env, code)
5767

5768 5769 5770 5771 5772 5773 5774
    def annotate(self, code):
        if self.exc_type:
            self.exc_type.annotate(code)
        if self.exc_value:
            self.exc_value.annotate(code)
        if self.exc_tb:
            self.exc_tb.annotate(code)
Haoyu Bai's avatar
Haoyu Bai committed
5775 5776
        if self.cause:
            self.cause.annotate(code)
5777

William Stein's avatar
William Stein committed
5778

5779 5780
class ReraiseStatNode(StatNode):

5781
    child_attrs = []
5782
    is_terminator = True
5783

5784
    def analyse_expressions(self, env):
5785
        return self
5786

5787
    nogil_check = Node.gil_error
5788 5789
    gil_message = "Raising exception"

5790
    def generate_execution_code(self, code):
5791
        code.mark_pos(self.pos)
5792
        vars = code.funcstate.exc_vars
5793
        if vars:
5794
            code.globalstate.use_utility_code(restore_exception_utility_code)
5795 5796 5797 5798
            code.put_giveref(vars[0])
            code.put_giveref(vars[1])
            # fresh exceptions may not have a traceback yet (-> finally!)
            code.put_xgiveref(vars[2])
5799
            code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % tuple(vars))
5800 5801 5802
            for varname in vars:
                code.put("%s = 0; " % varname)
            code.putln()
5803 5804
            code.putln(code.error_goto(self.pos))
        else:
5805 5806 5807
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ReRaiseException", "Exceptions.c"))
            code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos))
5808

William Stein's avatar
William Stein committed
5809 5810 5811 5812 5813
class AssertStatNode(StatNode):
    #  assert statement
    #
    #  cond    ExprNode
    #  value   ExprNode or None
5814

5815 5816
    child_attrs = ["cond", "value"]

William Stein's avatar
William Stein committed
5817 5818 5819
    def analyse_expressions(self, env):
        self.cond = self.cond.analyse_boolean_expression(env)
        if self.value:
5820
            value = self.value.analyse_types(env)
5821 5822
            if value.type is Builtin.tuple_type or not value.type.is_builtin_type:
                # prevent tuple values from being interpreted as argument value tuples
5823
                from .ExprNodes import TupleNode
5824
                value = TupleNode(value.pos, args=[value], slow=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
5825
                self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env)
5826 5827
            else:
                self.value = value.coerce_to_pyobject(env)
5828
        return self
5829

5830
    nogil_check = Node.gil_error
5831
    gil_message = "Raising exception"
5832

William Stein's avatar
William Stein committed
5833
    def generate_execution_code(self, code):
5834
        code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS")
5835
        code.putln("if (unlikely(!Py_OptimizeFlag)) {")
5836
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5837 5838
        self.cond.generate_evaluation_code(code)
        code.putln(
5839
            "if (unlikely(!%s)) {" % self.cond.result())
William Stein's avatar
William Stein committed
5840
        if self.value:
5841
            self.value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
5842
            code.putln(
5843
                "PyErr_SetObject(PyExc_AssertionError, %s);" % self.value.py_result())
5844
            self.value.generate_disposal_code(code)
5845
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
5846 5847 5848 5849
        else:
            code.putln(
                "PyErr_SetNone(PyExc_AssertionError);")
        code.putln(
5850
            code.error_goto(self.pos))
William Stein's avatar
William Stein committed
5851 5852 5853
        code.putln(
            "}")
        self.cond.generate_disposal_code(code)
5854
        self.cond.free_temps(code)
5855 5856
        code.putln(
            "}")
5857
        code.putln("#endif")
William Stein's avatar
William Stein committed
5858

5859 5860 5861 5862 5863
    def generate_function_definitions(self, env, code):
        self.cond.generate_function_definitions(env, code)
        if self.value is not None:
            self.value.generate_function_definitions(env, code)

5864 5865 5866 5867 5868 5869
    def annotate(self, code):
        self.cond.annotate(code)
        if self.value:
            self.value.annotate(code)


William Stein's avatar
William Stein committed
5870 5871 5872 5873 5874
class IfStatNode(StatNode):
    #  if statement
    #
    #  if_clauses   [IfClauseNode]
    #  else_clause  StatNode or None
5875 5876

    child_attrs = ["if_clauses", "else_clause"]
5877

William Stein's avatar
William Stein committed
5878 5879 5880 5881 5882
    def analyse_declarations(self, env):
        for if_clause in self.if_clauses:
            if_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
5883

William Stein's avatar
William Stein committed
5884
    def analyse_expressions(self, env):
5885
        self.if_clauses = [if_clause.analyse_expressions(env) for if_clause in self.if_clauses]
William Stein's avatar
William Stein committed
5886
        if self.else_clause:
5887 5888
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
5889

William Stein's avatar
William Stein committed
5890
    def generate_execution_code(self, code):
5891
        code.mark_pos(self.pos)
5892
        end_label = code.new_label()
5893 5894 5895
        last = len(self.if_clauses)
        if not self.else_clause:
            last -= 1  # avoid redundant goto at end of last if-clause
5896 5897
        for i, if_clause in enumerate(self.if_clauses):
            if_clause.generate_execution_code(code, end_label, is_last=i == last)
5898
        if self.else_clause:
5899
            code.mark_pos(self.else_clause.pos)
5900
            code.putln("/*else*/ {")
William Stein's avatar
William Stein committed
5901
            self.else_clause.generate_execution_code(code)
5902 5903
            code.putln("}")
        code.put_label(end_label)
5904

5905 5906 5907 5908 5909
    def generate_function_definitions(self, env, code):
        for clause in self.if_clauses:
            clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
5910

5911 5912 5913 5914 5915
    def annotate(self, code):
        for if_clause in self.if_clauses:
            if_clause.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
5916 5917 5918 5919 5920 5921 5922


class IfClauseNode(Node):
    #  if or elif clause in an if statement
    #
    #  condition   ExprNode
    #  body        StatNode
5923

5924 5925
    child_attrs = ["condition", "body"]

William Stein's avatar
William Stein committed
5926 5927
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
5928

William Stein's avatar
William Stein committed
5929
    def analyse_expressions(self, env):
5930
        self.condition = self.condition.analyse_temp_boolean_expression(env)
5931 5932
        self.body = self.body.analyse_expressions(env)
        return self
5933

5934
    def generate_execution_code(self, code, end_label, is_last):
William Stein's avatar
William Stein committed
5935
        self.condition.generate_evaluation_code(code)
5936 5937
        code.mark_pos(self.pos)
        code.putln("if (%s) {" % self.condition.result())
5938 5939
        self.condition.generate_disposal_code(code)
        self.condition.free_temps(code)
William Stein's avatar
William Stein committed
5940
        self.body.generate_execution_code(code)
5941 5942
        code.mark_pos(self.pos, trace=False)
        if not (is_last or self.body.is_terminator):
5943
            code.put_goto(end_label)
William Stein's avatar
William Stein committed
5944
        code.putln("}")
5945

5946 5947 5948 5949
    def generate_function_definitions(self, env, code):
        self.condition.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)

5950 5951 5952
    def annotate(self, code):
        self.condition.annotate(code)
        self.body.annotate(code)
5953

5954 5955 5956 5957 5958 5959

class SwitchCaseNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # conditions    [ExprNode]
    # body          StatNode
5960

5961
    child_attrs = ['conditions', 'body']
5962

5963 5964
    def generate_execution_code(self, code):
        for cond in self.conditions:
5965
            code.mark_pos(cond.pos)
5966 5967
            cond.generate_evaluation_code(code)
            code.putln("case %s:" % cond.result())
5968
        self.body.generate_execution_code(code)
5969
        code.mark_pos(self.pos, trace=False)
5970
        code.putln("break;")
5971 5972 5973 5974 5975

    def generate_function_definitions(self, env, code):
        for cond in self.conditions:
            cond.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
5976

5977 5978 5979
    def annotate(self, code):
        for cond in self.conditions:
            cond.annotate(code)
5980
        self.body.annotate(code)
5981

5982

5983 5984 5985 5986 5987 5988
class SwitchStatNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # test          ExprNode
    # cases         [SwitchCaseNode]
    # else_clause   StatNode or None
5989

5990
    child_attrs = ['test', 'cases', 'else_clause']
5991

5992
    def generate_execution_code(self, code):
5993
        self.test.generate_evaluation_code(code)
5994
        code.mark_pos(self.pos)
5995
        code.putln("switch (%s) {" % self.test.result())
5996 5997 5998 5999 6000
        for case in self.cases:
            case.generate_execution_code(code)
        if self.else_clause is not None:
            code.putln("default:")
            self.else_clause.generate_execution_code(code)
6001
            code.putln("break;")
6002 6003 6004 6005 6006
        else:
            # Always generate a default clause to prevent C compiler warnings
            # about unmatched enum values (it was not the user who decided to
            # generate the switch statement, so shouldn't be bothered).
            code.putln("default: break;")
6007 6008
        code.putln("}")

6009 6010 6011 6012 6013 6014 6015
    def generate_function_definitions(self, env, code):
        self.test.generate_function_definitions(env, code)
        for case in self.cases:
            case.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6016 6017 6018 6019
    def annotate(self, code):
        self.test.annotate(code)
        for case in self.cases:
            case.annotate(code)
6020 6021
        if self.else_clause is not None:
            self.else_clause.annotate(code)
6022

6023

6024
class LoopNode(object):
6025
    pass
6026

6027

6028
class WhileStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6029 6030 6031 6032 6033
    #  while statement
    #
    #  condition    ExprNode
    #  body         StatNode
    #  else_clause  StatNode
6034 6035

    child_attrs = ["condition", "body", "else_clause"]
6036

William Stein's avatar
William Stein committed
6037 6038 6039 6040
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6041

William Stein's avatar
William Stein committed
6042
    def analyse_expressions(self, env):
6043 6044
        if self.condition:
            self.condition = self.condition.analyse_temp_boolean_expression(env)
6045
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6046
        if self.else_clause:
6047 6048
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6049

William Stein's avatar
William Stein committed
6050
    def generate_execution_code(self, code):
6051
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6052 6053 6054
        old_loop_labels = code.new_loop_labels()
        code.putln(
            "while (1) {")
6055 6056 6057 6058
        if self.condition:
            self.condition.generate_evaluation_code(code)
            self.condition.generate_disposal_code(code)
            code.putln(
6059
                "if (!%s) break;" % self.condition.result())
6060
            self.condition.free_temps(code)
William Stein's avatar
William Stein committed
6061
        self.body.generate_execution_code(code)
6062
        code.put_label(code.continue_label)
William Stein's avatar
William Stein committed
6063 6064 6065 6066
        code.putln("}")
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
6067
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
6068 6069 6070 6071 6072
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)

6073
    def generate_function_definitions(self, env, code):
6074 6075
        if self.condition:
            self.condition.generate_function_definitions(env, code)
6076 6077 6078 6079
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6080
    def annotate(self, code):
6081 6082
        if self.condition:
            self.condition.annotate(code)
6083 6084 6085 6086
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
6087

6088 6089 6090 6091
class DictIterationNextNode(Node):
    # Helper node for calling PyDict_Next() inside of a WhileStatNode
    # and checking the dictionary size for changes.  Created in
    # Optimize.py.
6092 6093 6094
    child_attrs = ['dict_obj', 'expected_size', 'pos_index_var',
                   'coerced_key_var', 'coerced_value_var', 'coerced_tuple_var',
                   'key_target', 'value_target', 'tuple_target', 'is_dict_flag']
6095

6096 6097 6098 6099 6100 6101
    coerced_key_var = key_ref = None
    coerced_value_var = value_ref = None
    coerced_tuple_var = tuple_ref = None

    def __init__(self, dict_obj, expected_size, pos_index_var,
                 key_target, value_target, tuple_target, is_dict_flag):
6102 6103
        Node.__init__(
            self, dict_obj.pos,
6104 6105 6106 6107 6108 6109 6110 6111 6112
            dict_obj=dict_obj,
            expected_size=expected_size,
            pos_index_var=pos_index_var,
            key_target=key_target,
            value_target=value_target,
            tuple_target=tuple_target,
            is_dict_flag=is_dict_flag,
            is_temp=True,
            type=PyrexTypes.c_bint_type)
6113 6114

    def analyse_expressions(self, env):
6115
        from . import ExprNodes
6116 6117 6118 6119
        self.dict_obj = self.dict_obj.analyse_types(env)
        self.expected_size = self.expected_size.analyse_types(env)
        if self.pos_index_var:
            self.pos_index_var = self.pos_index_var.analyse_types(env)
6120
        if self.key_target:
6121
            self.key_target = self.key_target.analyse_target_types(env)
6122
            self.key_ref = ExprNodes.TempNode(self.key_target.pos, PyrexTypes.py_object_type)
6123 6124
            self.coerced_key_var = self.key_ref.coerce_to(self.key_target.type, env)
        if self.value_target:
6125
            self.value_target = self.value_target.analyse_target_types(env)
6126
            self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
6127 6128
            self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
        if self.tuple_target:
6129
            self.tuple_target = self.tuple_target.analyse_target_types(env)
6130
            self.tuple_ref = ExprNodes.TempNode(self.tuple_target.pos, PyrexTypes.py_object_type)
6131
            self.coerced_tuple_var = self.tuple_ref.coerce_to(self.tuple_target.type, env)
6132 6133
        self.is_dict_flag = self.is_dict_flag.analyse_types(env)
        return self
6134 6135 6136 6137 6138

    def generate_function_definitions(self, env, code):
        self.dict_obj.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
6139
        code.globalstate.use_utility_code(UtilityCode.load_cached("dict_iter", "Optimize.c"))
6140 6141
        self.dict_obj.generate_evaluation_code(code)

6142 6143 6144 6145 6146 6147 6148 6149
        assignments = []
        temp_addresses = []
        for var, result, target in [(self.key_ref, self.coerced_key_var, self.key_target),
                                    (self.value_ref, self.coerced_value_var, self.value_target),
                                    (self.tuple_ref, self.coerced_tuple_var, self.tuple_target)]:
            if target is None:
                addr = 'NULL'
            else:
6150 6151 6152
                assignments.append((var, result, target))
                var.allocate(code)
                addr = '&%s' % var.result()
6153 6154 6155 6156 6157
            temp_addresses.append(addr)

        result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
        code.putln("%s = __Pyx_dict_iter_next(%s, %s, &%s, %s, %s, %s, %s);" % (
            result_temp,
6158
            self.dict_obj.py_result(),
6159 6160 6161 6162 6163 6164 6165 6166 6167 6168 6169 6170
            self.expected_size.result(),
            self.pos_index_var.result(),
            temp_addresses[0],
            temp_addresses[1],
            temp_addresses[2],
            self.is_dict_flag.result()
        ))
        code.putln("if (unlikely(%s == 0)) break;" % result_temp)
        code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
        code.funcstate.release_temp(result_temp)

        # evaluate all coercions before the assignments
6171 6172
        for var, result, target in assignments:
            code.put_gotref(var.result())
6173
        for var, result, target in assignments:
6174
            result.generate_evaluation_code(code)
6175
        for var, result, target in assignments:
6176
            target.generate_assignment_code(result, code)
6177
            var.release(code)
6178

6179

Robert Bradshaw's avatar
Robert Bradshaw committed
6180
def ForStatNode(pos, **kw):
6181
    if 'iterator' in kw:
6182 6183 6184 6185
        if kw['iterator'].is_async:
            return AsyncForStatNode(pos, **kw)
        else:
            return ForInStatNode(pos, **kw)
Robert Bradshaw's avatar
Robert Bradshaw committed
6186 6187 6188
    else:
        return ForFromStatNode(pos, **kw)

6189 6190 6191

class _ForInStatNode(LoopNode, StatNode):
    #  Base class of 'for-in' statements.
William Stein's avatar
William Stein committed
6192 6193
    #
    #  target        ExprNode
6194
    #  iterator      IteratorNode | AIterAwaitExprNode(AsyncIteratorNode)
William Stein's avatar
William Stein committed
6195 6196
    #  body          StatNode
    #  else_clause   StatNode
6197 6198
    #  item          NextNode | AwaitExprNode(AsyncNextNode)
    #  is_async      boolean        true for 'async for' statements
6199

6200
    child_attrs = ["target", "item", "iterator", "body", "else_clause"]
6201
    item = None
6202 6203 6204 6205
    is_async = False

    def _create_item_node(self):
        raise NotImplementedError("must be implemented by subclasses")
6206

William Stein's avatar
William Stein committed
6207 6208 6209 6210 6211
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6212
        self._create_item_node()
6213

William Stein's avatar
William Stein committed
6214
    def analyse_expressions(self, env):
6215
        self.target = self.target.analyse_target_types(env)
6216
        self.iterator = self.iterator.analyse_expressions(env)
6217
        self._create_item_node()  # must rewrap self.item after analysis
6218
        self.item = self.item.analyse_expressions(env)
6219 6220 6221
        if (not self.is_async and
                (self.iterator.type.is_ptr or self.iterator.type.is_array) and
                self.target.type.assignable_from(self.iterator.type)):
6222 6223 6224 6225
            # C array slice optimization.
            pass
        else:
            self.item = self.item.coerce_to(self.target.type, env)
6226
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6227
        if self.else_clause:
6228 6229
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
William Stein's avatar
William Stein committed
6230 6231

    def generate_execution_code(self, code):
6232
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6233 6234
        old_loop_labels = code.new_loop_labels()
        self.iterator.generate_evaluation_code(code)
Mark Florisson's avatar
Mark Florisson committed
6235
        code.putln("for (;;) {")
William Stein's avatar
William Stein committed
6236 6237 6238
        self.item.generate_evaluation_code(code)
        self.target.generate_assignment_code(self.item, code)
        self.body.generate_execution_code(code)
6239
        code.mark_pos(self.pos)
6240
        code.put_label(code.continue_label)
Mark Florisson's avatar
Mark Florisson committed
6241
        code.putln("}")
William Stein's avatar
William Stein committed
6242 6243
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
6244

William Stein's avatar
William Stein committed
6245
        if self.else_clause:
6246 6247 6248 6249 6250 6251 6252
            # in nested loops, the 'else' block can contain a
            # 'continue' statement for the outer loop, but we may need
            # to generate cleanup code before taking that path, so we
            # intercept it here
            orig_continue_label = code.continue_label
            code.continue_label = code.new_label('outer_continue')

William Stein's avatar
William Stein committed
6253 6254 6255
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
6256 6257 6258

            if code.label_used(code.continue_label):
                code.put_goto(break_label)
6259
                code.mark_pos(self.pos)
6260 6261 6262 6263 6264
                code.put_label(code.continue_label)
                self.iterator.generate_disposal_code(code)
                code.put_goto(orig_continue_label)
            code.set_loop_labels(old_loop_labels)

6265
        code.mark_pos(self.pos)
6266 6267
        if code.label_used(break_label):
            code.put_label(break_label)
William Stein's avatar
William Stein committed
6268
        self.iterator.generate_disposal_code(code)
6269
        self.iterator.free_temps(code)
William Stein's avatar
William Stein committed
6270

6271 6272 6273 6274 6275 6276 6277
    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.iterator.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6278 6279 6280 6281 6282 6283 6284 6285
    def annotate(self, code):
        self.target.annotate(code)
        self.iterator.annotate(code)
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
        self.item.annotate(code)

William Stein's avatar
William Stein committed
6286

6287 6288 6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299
class ForInStatNode(_ForInStatNode):
    #  'for' statement

    is_async = False

    def _create_item_node(self):
        from .ExprNodes import NextNode
        self.item = NextNode(self.iterator)


class AsyncForStatNode(_ForInStatNode):
    #  'async for' statement
    #
6300
    #  iterator      AIterAwaitExprNode(AsyncIteratorNode)
6301 6302 6303 6304 6305 6306 6307 6308
    #  item          AwaitIterNextExprNode(AsyncIteratorNode)

    is_async = True

    def __init__(self, pos, iterator, **kw):
        assert 'item' not in kw
        from . import ExprNodes
        # AwaitExprNodes must appear before running MarkClosureVisitor
6309
        kw['iterator'] = ExprNodes.AIterAwaitExprNode(iterator.pos, arg=iterator)
6310 6311 6312 6313 6314 6315 6316 6317
        kw['item'] = ExprNodes.AwaitIterNextExprNode(iterator.pos, arg=None)
        _ForInStatNode.__init__(self, pos, **kw)

    def _create_item_node(self):
        from . import ExprNodes
        self.item.arg = ExprNodes.AsyncNextNode(self.iterator)


6318
class ForFromStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6319 6320 6321 6322 6323 6324 6325
    #  for name from expr rel name rel expr
    #
    #  target        NameNode
    #  bound1        ExprNode
    #  relation1     string
    #  relation2     string
    #  bound2        ExprNode
6326
    #  step          ExprNode or None
William Stein's avatar
William Stein committed
6327 6328 6329 6330 6331
    #  body          StatNode
    #  else_clause   StatNode or None
    #
    #  Used internally:
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
6332
    #  from_range         bool
6333
    #  is_py_target       bool
6334
    #  loopvar_node       ExprNode (usually a NameNode or temp node)
William Stein's avatar
William Stein committed
6335
    #  py_loopvar_node    PyTempNode or None
6336
    child_attrs = ["target", "bound1", "bound2", "step", "body", "else_clause"]
6337 6338

    is_py_target = False
6339
    loopvar_node = None
6340
    py_loopvar_node = None
Robert Bradshaw's avatar
Robert Bradshaw committed
6341
    from_range = False
6342

6343 6344 6345 6346 6347 6348 6349
    gil_message = "For-loop using object bounds or target"

    def nogil_check(self, env):
        for x in (self.target, self.bound1, self.bound2):
            if x.type.is_pyobject:
                self.gil_error()

Robert Bradshaw's avatar
Robert Bradshaw committed
6350 6351 6352 6353 6354
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6355

William Stein's avatar
William Stein committed
6356
    def analyse_expressions(self, env):
6357
        from . import ExprNodes
6358 6359 6360
        self.target = self.target.analyse_target_types(env)
        self.bound1 = self.bound1.analyse_types(env)
        self.bound2 = self.bound2.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6361 6362
        if self.step is not None:
            if isinstance(self.step, ExprNodes.UnaryMinusNode):
6363 6364
                warning(self.step.pos, "Probable infinite loop in for-from-by statement. "
                        "Consider switching the directions of the relations.", 2)
6365
            self.step = self.step.analyse_types(env)
6366

6367
        if self.target.type.is_numeric:
Robert Bradshaw's avatar
Robert Bradshaw committed
6368
            loop_type = self.target.type
6369
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6370 6371 6372 6373 6374 6375 6376 6377 6378
            loop_type = PyrexTypes.c_int_type
            if not self.bound1.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound1.type)
            if not self.bound2.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound2.type)
            if self.step is not None and not self.step.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.step.type)
        self.bound1 = self.bound1.coerce_to(loop_type, env)
        self.bound2 = self.bound2.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6379 6380
        if not self.bound2.is_literal:
            self.bound2 = self.bound2.coerce_to_temp(env)
6381
        if self.step is not None:
6382
            self.step = self.step.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6383 6384
            if not self.step.is_literal:
                self.step = self.step.coerce_to_temp(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6385

William Stein's avatar
William Stein committed
6386
        target_type = self.target.type
6387
        if not (target_type.is_pyobject or target_type.is_numeric):
6388
            error(self.target.pos, "for-from loop variable must be c numeric type or Python object")
6389
        if target_type.is_numeric:
Robert Bradshaw's avatar
Robert Bradshaw committed
6390
            self.is_py_target = False
6391 6392
            if isinstance(self.target, ExprNodes.BufferIndexNode):
                raise error(self.pos, "Buffer or memoryview slicing/indexing not allowed as for-loop target.")
6393
            self.loopvar_node = self.target
William Stein's avatar
William Stein committed
6394 6395
            self.py_loopvar_node = None
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6396 6397
            self.is_py_target = True
            c_loopvar_node = ExprNodes.TempNode(self.pos, loop_type, env)
6398
            self.loopvar_node = c_loopvar_node
William Stein's avatar
William Stein committed
6399 6400
            self.py_loopvar_node = \
                ExprNodes.CloneNode(c_loopvar_node).coerce_to_pyobject(env)
6401
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6402
        if self.else_clause:
6403 6404
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6405

William Stein's avatar
William Stein committed
6406
    def generate_execution_code(self, code):
6407
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
6408
        old_loop_labels = code.new_loop_labels()
Robert Bradshaw's avatar
Robert Bradshaw committed
6409
        from_range = self.from_range
William Stein's avatar
William Stein committed
6410 6411 6412
        self.bound1.generate_evaluation_code(code)
        self.bound2.generate_evaluation_code(code)
        offset, incop = self.relation_table[self.relation1]
6413 6414
        if self.step is not None:
            self.step.generate_evaluation_code(code)
Magnus Lie Hetland's avatar
Magnus Lie Hetland committed
6415 6416
            step = self.step.result()
            incop = "%s=%s" % (incop[0], step)
6417
        from . import ExprNodes
6418 6419 6420 6421
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.allocate(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.allocate(code)
6422
        if from_range:
Robert Bradshaw's avatar
Robert Bradshaw committed
6423
            loopvar_name = code.funcstate.allocate_temp(self.target.type, False)
6424
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6425
            loopvar_name = self.loopvar_node.result()
6426 6427 6428 6429 6430
        if self.target.type.is_int and not self.target.type.signed and self.relation2[0] == '>':
            # Handle the case where the endpoint of an unsigned int iteration
            # is within step of 0.
            if not self.step:
                step = 1
6431 6432 6433 6434 6435
            code.putln("for (%s = %s%s + %s; %s %s %s + %s; ) { %s%s;" % (
                loopvar_name,
                self.bound1.result(), offset, step,
                loopvar_name, self.relation2, self.bound2.result(), step,
                loopvar_name, incop))
6436
        else:
6437 6438 6439 6440 6441
            code.putln("for (%s = %s%s; %s %s %s; %s%s) {" % (
                loopvar_name,
                self.bound1.result(), offset,
                loopvar_name, self.relation2, self.bound2.result(),
                loopvar_name, incop))
William Stein's avatar
William Stein committed
6442 6443 6444
        if self.py_loopvar_node:
            self.py_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(self.py_loopvar_node, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
6445 6446
        elif from_range:
            code.putln("%s = %s;" % (
6447
                self.target.result(), loopvar_name))
William Stein's avatar
William Stein committed
6448 6449
        self.body.generate_execution_code(code)
        code.put_label(code.continue_label)
Robert Bradshaw's avatar
Robert Bradshaw committed
6450
        if self.py_loopvar_node:
6451 6452 6453
            # This mess is to make for..from loops with python targets behave
            # exactly like those with C targets with regards to re-assignment
            # of the loop variable.
6454
            if self.target.entry.is_pyglobal:
6455
                # We know target is a NameNode, this is the only ugly case.
6456
                target_node = ExprNodes.PyTempNode(self.target.pos, None)
6457 6458
                target_node.allocate(code)
                interned_cname = code.intern_identifier(self.target.entry.name)
6459 6460 6461 6462 6463 6464 6465 6466 6467 6468
                if self.target.entry.scope.is_module_scope:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c"))
                    lookup_func = '__Pyx_GetModuleGlobalName(%s)'
                else:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetNameInClass", "ObjectHandling.c"))
                    lookup_func = '__Pyx_GetNameInClass(%s, %%s)' % (
                        self.target.entry.scope.namespace_cname)
                code.putln("%s = %s; %s" % (
6469
                    target_node.result(),
6470
                    lookup_func % interned_cname,
6471
                    code.error_goto_if_null(target_node.result(), self.target.pos)))
6472
                code.put_gotref(target_node.result())
6473 6474
            else:
                target_node = self.target
6475 6476
            from_py_node = ExprNodes.CoerceFromPyTypeNode(
                self.loopvar_node.type, target_node, self.target.entry.scope)
6477 6478
            from_py_node.temp_code = loopvar_name
            from_py_node.generate_result_code(code)
6479
            if self.target.entry.is_pyglobal:
6480 6481
                code.put_decref(target_node.result(), target_node.type)
                target_node.release(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
6482
        code.putln("}")
6483
        if self.py_loopvar_node:
6484 6485
            # This is potentially wasteful, but we don't want the semantics to
            # depend on whether or not the loop is a python type.
6486 6487
            self.py_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(self.py_loopvar_node, code)
6488 6489
        if from_range:
            code.funcstate.release_temp(loopvar_name)
William Stein's avatar
William Stein committed
6490 6491 6492 6493 6494 6495 6496 6497
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)
        self.bound1.generate_disposal_code(code)
6498
        self.bound1.free_temps(code)
William Stein's avatar
William Stein committed
6499
        self.bound2.generate_disposal_code(code)
6500
        self.bound2.free_temps(code)
6501 6502 6503 6504
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.release(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.release(code)
6505 6506
        if self.step is not None:
            self.step.generate_disposal_code(code)
6507
            self.step.free_temps(code)
6508

William Stein's avatar
William Stein committed
6509 6510 6511 6512 6513
    relation_table = {
        # {relop : (initial offset, increment op)}
        '<=': ("",   "++"),
        '<' : ("+1", "++"),
        '>=': ("",   "--"),
6514
        '>' : ("-1", "--"),
William Stein's avatar
William Stein committed
6515
    }
6516 6517 6518 6519 6520 6521 6522 6523 6524 6525

    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.bound1.generate_function_definitions(env, code)
        self.bound2.generate_function_definitions(env, code)
        if self.step is not None:
            self.step.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
6526

6527 6528 6529 6530 6531
    def annotate(self, code):
        self.target.annotate(code)
        self.bound1.annotate(code)
        self.bound2.annotate(code)
        if self.step:
6532
            self.step.annotate(code)
6533 6534 6535
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
6536 6537


6538 6539 6540
class WithStatNode(StatNode):
    """
    Represents a Python with statement.
6541

6542
    Implemented by the WithTransform as follows:
6543 6544 6545 6546 6547 6548 6549 6550 6551 6552 6553 6554 6555 6556 6557 6558 6559

        MGR = EXPR
        EXIT = MGR.__exit__
        VALUE = MGR.__enter__()
        EXC = True
        try:
            try:
                TARGET = VALUE  # optional
                BODY
            except:
                EXC = False
                if not EXIT(*EXCINFO):
                    raise
        finally:
            if EXC:
                EXIT(None, None, None)
            MGR = EXIT = VALUE = None
6560 6561
    """
    #  manager          The with statement manager object
6562
    #  target           ExprNode  the target lhs of the __enter__() call
6563
    #  body             StatNode
6564
    #  enter_call       ExprNode  the call to the __enter__() method
6565
    #  exit_var         String    the cname of the __exit__() method reference
6566

6567
    child_attrs = ["manager", "enter_call", "target", "body"]
6568

6569
    enter_call = None
6570
    target_temp = None
6571 6572 6573

    def analyse_declarations(self, env):
        self.manager.analyse_declarations(env)
6574
        self.enter_call.analyse_declarations(env)
6575 6576 6577
        self.body.analyse_declarations(env)

    def analyse_expressions(self, env):
6578 6579
        self.manager = self.manager.analyse_types(env)
        self.enter_call = self.enter_call.analyse_types(env)
6580 6581 6582 6583
        if self.target:
            # set up target_temp before descending into body (which uses it)
            from .ExprNodes import TempNode
            self.target_temp = TempNode(self.enter_call.pos, self.enter_call.type)
6584 6585
        self.body = self.body.analyse_expressions(env)
        return self
6586

6587 6588
    def generate_function_definitions(self, env, code):
        self.manager.generate_function_definitions(env, code)
6589
        self.enter_call.generate_function_definitions(env, code)
6590 6591
        self.body.generate_function_definitions(env, code)

6592
    def generate_execution_code(self, code):
6593
        code.mark_pos(self.pos)
6594 6595 6596
        code.putln("/*with:*/ {")
        self.manager.generate_evaluation_code(code)
        self.exit_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
6597
        code.globalstate.use_utility_code(
6598 6599
            UtilityCode.load_cached("PyObjectLookupSpecial", "ObjectHandling.c"))
        code.putln("%s = __Pyx_PyObject_LookupSpecial(%s, %s); %s" % (
6600 6601
            self.exit_var,
            self.manager.py_result(),
6602
            code.intern_identifier(EncodedString('__aexit__' if self.is_async else '__exit__')),
6603 6604 6605 6606 6607 6608 6609 6610
            code.error_goto_if_null(self.exit_var, self.pos),
            ))
        code.put_gotref(self.exit_var)

        # need to free exit_var in the face of exceptions during setup
        old_error_label = code.new_error_label()
        intermediate_error_label = code.error_label

6611
        self.enter_call.generate_evaluation_code(code)
6612 6613 6614 6615 6616 6617 6618
        if self.target:
            # The temp result will be cleaned up by the WithTargetAssignmentStatNode
            # after assigning its result to the target of the 'with' statement.
            self.target_temp.allocate(code)
            self.enter_call.make_owned_reference(code)
            code.putln("%s = %s;" % (self.target_temp.result(), self.enter_call.result()))
            self.enter_call.generate_post_assignment_code(code)
6619
        else:
6620 6621 6622
            self.enter_call.generate_disposal_code(code)
        self.enter_call.free_temps(code)

6623 6624 6625 6626 6627 6628
        self.manager.generate_disposal_code(code)
        self.manager.free_temps(code)

        code.error_label = old_error_label
        self.body.generate_execution_code(code)

6629 6630 6631 6632 6633 6634 6635
        if code.label_used(intermediate_error_label):
            step_over_label = code.new_label()
            code.put_goto(step_over_label)
            code.put_label(intermediate_error_label)
            code.put_decref_clear(self.exit_var, py_object_type)
            code.put_goto(old_error_label)
            code.put_label(step_over_label)
6636 6637 6638 6639

        code.funcstate.release_temp(self.exit_var)
        code.putln('}')

6640

6641 6642 6643 6644
class WithTargetAssignmentStatNode(AssignmentNode):
    # The target assignment of the 'with' statement value (return
    # value of the __enter__() call).
    #
6645
    # This is a special cased assignment that properly cleans up the RHS.
6646
    #
6647 6648 6649
    # lhs       ExprNode      the assignment target
    # rhs       ExprNode      a (coerced) TempNode for the rhs (from WithStatNode)
    # with_node WithStatNode  the surrounding with-statement
6650

6651 6652 6653
    child_attrs = ["rhs", "lhs"]
    with_node = None
    rhs = None
6654 6655 6656 6657

    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)

6658
    def analyse_expressions(self, env):
6659
        self.lhs = self.lhs.analyse_target_types(env)
6660
        self.lhs.gil_assignment_check(env)
6661
        self.rhs = self.with_node.target_temp.coerce_to(self.lhs.type, env)
6662
        return self
6663 6664 6665 6666

    def generate_execution_code(self, code):
        self.rhs.generate_evaluation_code(code)
        self.lhs.generate_assignment_code(self.rhs, code)
6667
        self.with_node.target_temp.release(code)
6668 6669 6670 6671 6672

    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

6673

William Stein's avatar
William Stein committed
6674 6675 6676 6677 6678 6679
class TryExceptStatNode(StatNode):
    #  try .. except statement
    #
    #  body             StatNode
    #  except_clauses   [ExceptClauseNode]
    #  else_clause      StatNode or None
6680

6681
    child_attrs = ["body", "except_clauses", "else_clause"]
6682

William Stein's avatar
William Stein committed
6683 6684 6685 6686 6687 6688
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        for except_clause in self.except_clauses:
            except_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6689

William Stein's avatar
William Stein committed
6690
    def analyse_expressions(self, env):
6691
        self.body = self.body.analyse_expressions(env)
6692
        default_clause_seen = 0
6693 6694
        for i, except_clause in enumerate(self.except_clauses):
            except_clause = self.except_clauses[i] = except_clause.analyse_expressions(env)
6695 6696 6697 6698 6699
            if default_clause_seen:
                error(except_clause.pos, "default 'except:' must be last")
            if not except_clause.pattern:
                default_clause_seen = 1
        self.has_default_clause = default_clause_seen
William Stein's avatar
William Stein committed
6700
        if self.else_clause:
6701 6702
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6703

6704
    nogil_check = Node.gil_error
6705 6706
    gil_message = "Try-except statement"

William Stein's avatar
William Stein committed
6707
    def generate_execution_code(self, code):
6708
        old_return_label = code.return_label
6709
        old_break_label = code.break_label
6710
        old_continue_label = code.continue_label
William Stein's avatar
William Stein committed
6711 6712
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
6713 6714 6715
        except_end_label = code.new_label('exception_handled')
        except_error_label = code.new_label('except_error')
        except_return_label = code.new_label('except_return')
6716
        try_return_label = code.new_label('try_return')
6717 6718
        try_break_label = code.new_label('try_break') if old_break_label else None
        try_continue_label = code.new_label('try_continue') if old_continue_label else None
6719
        try_end_label = code.new_label('try_end')
6720

6721
        exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
6722
                         for _ in range(3)]
6723
        code.mark_pos(self.pos)
6724
        code.putln("{")
6725
        save_exc = code.insertion_point()
William Stein's avatar
William Stein committed
6726 6727
        code.putln(
            "/*try:*/ {")
6728
        code.return_label = try_return_label
6729
        code.break_label = try_break_label
6730
        code.continue_label = try_continue_label
William Stein's avatar
William Stein committed
6731
        self.body.generate_execution_code(code)
6732
        code.mark_pos(self.pos, trace=False)
William Stein's avatar
William Stein committed
6733 6734
        code.putln(
            "}")
6735
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
6736 6737 6738 6739
        can_raise = code.label_used(our_error_label)

        if can_raise:
            # inject code before the try block to save away the exception state
6740
            code.globalstate.use_utility_code(reset_exception_utility_code)
6741 6742 6743 6744
            save_exc.putln("__Pyx_PyThreadState_declare")
            save_exc.putln("__Pyx_PyThreadState_assign")
            save_exc.putln("__Pyx_ExceptionSave(%s);" % (
                ', '.join(['&%s' % var for var in exc_save_vars])))
6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755
            for var in exc_save_vars:
                save_exc.put_xgotref(var)

            def restore_saved_exception():
                for name in exc_save_vars:
                    code.put_xgiveref(name)
                code.putln("__Pyx_ExceptionReset(%s);" %
                           ', '.join(exc_save_vars))
        else:
            # try block cannot raise exceptions, but we had to allocate the temps above,
            # so just keep the C compiler from complaining about them being unused
6756
            save_exc.putln("if (%s); else {/*mark used*/}" % '||'.join(exc_save_vars))
6757 6758 6759 6760

            def restore_saved_exception():
                pass

6761 6762
        code.error_label = except_error_label
        code.return_label = except_return_label
6763
        normal_case_terminates = self.body.is_terminator
William Stein's avatar
William Stein committed
6764
        if self.else_clause:
6765
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
6766 6767 6768 6769 6770
            code.putln(
                "/*else:*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln(
                "}")
6771 6772
            if not normal_case_terminates:
                normal_case_terminates = self.else_clause.is_terminator
6773

6774
        if can_raise:
6775 6776 6777 6778
            if not normal_case_terminates:
                for var in exc_save_vars:
                    code.put_xdecref_clear(var, py_object_type)
                code.put_goto(try_end_label)
6779
            code.put_label(our_error_label)
6780
            code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
6781 6782 6783 6784 6785 6786 6787 6788 6789 6790 6791 6792
            for temp_name, temp_type in temps_to_clean_up:
                code.put_xdecref_clear(temp_name, temp_type)
            for except_clause in self.except_clauses:
                except_clause.generate_handling_code(code, except_end_label)
            if not self.has_default_clause:
                code.put_goto(except_error_label)

        for exit_label, old_label in [(except_error_label, old_error_label),
                                      (try_break_label, old_break_label),
                                      (try_continue_label, old_continue_label),
                                      (try_return_label, old_return_label),
                                      (except_return_label, old_return_label)]:
6793
            if code.label_used(exit_label):
6794
                if not normal_case_terminates and not code.label_used(try_end_label):
6795
                    code.put_goto(try_end_label)
6796
                code.put_label(exit_label)
6797
                code.mark_pos(self.pos, trace=False)
6798 6799 6800
                if can_raise:
                    code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
                    restore_saved_exception()
6801
                code.put_goto(old_label)
6802 6803

        if code.label_used(except_end_label):
6804
            if not normal_case_terminates and not code.label_used(try_end_label):
6805
                code.put_goto(try_end_label)
6806
            code.put_label(except_end_label)
6807 6808 6809
            if can_raise:
                code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
                restore_saved_exception()
6810 6811
        if code.label_used(try_end_label):
            code.put_label(try_end_label)
6812 6813
        code.putln("}")

6814 6815 6816
        for cname in exc_save_vars:
            code.funcstate.release_temp(cname)

6817
        code.return_label = old_return_label
6818
        code.break_label = old_break_label
6819
        code.continue_label = old_continue_label
6820
        code.error_label = old_error_label
William Stein's avatar
William Stein committed
6821

6822 6823 6824 6825 6826 6827 6828
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        for except_clause in self.except_clauses:
            except_clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6829 6830 6831 6832 6833 6834 6835
    def annotate(self, code):
        self.body.annotate(code)
        for except_node in self.except_clauses:
            except_node.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
6836 6837 6838 6839

class ExceptClauseNode(Node):
    #  Part of try ... except statement.
    #
6840
    #  pattern        [ExprNode]
William Stein's avatar
William Stein committed
6841 6842
    #  target         ExprNode or None
    #  body           StatNode
6843
    #  excinfo_target TupleNode(3*ResultRefNode) or None   optional target for exception info (not owned here!)
William Stein's avatar
William Stein committed
6844 6845 6846
    #  match_flag     string             result of exception match
    #  exc_value      ExcValueNode       used internally
    #  function_name  string             qualified name of enclosing function
6847
    #  exc_vars       (string * 3)       local exception variables
6848
    #  is_except_as   bool               Py3-style "except ... as xyz"
6849 6850 6851 6852

    # excinfo_target is never set by the parser, but can be set by a transform
    # in order to extract more extensive information about the exception as a
    # sys.exc_info()-style tuple into a target variable
6853

6854
    child_attrs = ["pattern", "target", "body", "exc_value"]
6855

6856
    exc_value = None
6857
    excinfo_target = None
6858
    is_except_as = False
6859

William Stein's avatar
William Stein committed
6860 6861 6862 6863
    def analyse_declarations(self, env):
        if self.target:
            self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
6864

William Stein's avatar
William Stein committed
6865 6866 6867
    def analyse_expressions(self, env):
        self.function_name = env.qualified_name
        if self.pattern:
6868 6869
            # normalise/unpack self.pattern into a list
            for i, pattern in enumerate(self.pattern):
6870
                pattern = pattern.analyse_expressions(env)
6871
                self.pattern[i] = pattern.coerce_to_pyobject(env)
6872

William Stein's avatar
William Stein committed
6873
        if self.target:
6874
            from . import ExprNodes
6875
            self.exc_value = ExprNodes.ExcValueNode(self.pos)
6876
            self.target = self.target.analyse_target_expression(env, self.exc_value)
6877

6878 6879
        self.body = self.body.analyse_expressions(env)
        return self
6880

William Stein's avatar
William Stein committed
6881 6882 6883
    def generate_handling_code(self, code, end_label):
        code.mark_pos(self.pos)
        if self.pattern:
6884
            code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrExceptionMatches", "Exceptions.c"))
6885 6886 6887
            exc_tests = []
            for pattern in self.pattern:
                pattern.generate_evaluation_code(code)
6888
                exc_tests.append("__Pyx_PyErr_ExceptionMatches(%s)" % pattern.py_result())
6889

6890
            match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
William Stein's avatar
William Stein committed
6891
            code.putln(
6892 6893 6894 6895
                "%s = %s;" % (match_flag, ' || '.join(exc_tests)))
            for pattern in self.pattern:
                pattern.generate_disposal_code(code)
                pattern.free_temps(code)
William Stein's avatar
William Stein committed
6896 6897
            code.putln(
                "if (%s) {" %
6898 6899
                    match_flag)
            code.funcstate.release_temp(match_flag)
William Stein's avatar
William Stein committed
6900
        else:
6901
            code.putln("/*except:*/ {")
6902

6903 6904
        if (not getattr(self.body, 'stats', True)
                and self.excinfo_target is None
6905
                and self.target is None):
Stefan Behnel's avatar
Stefan Behnel committed
6906 6907
            # most simple case: no exception variable, empty body (pass)
            # => reset the exception state, done
6908 6909
            code.globalstate.use_utility_code(UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c"))
            code.putln("__Pyx_ErrRestore(0,0,0);")
6910 6911 6912
            code.put_goto(end_label)
            code.putln("}")
            return
6913

6914 6915
        exc_vars = [code.funcstate.allocate_temp(py_object_type,
                                                 manage_ref=True)
6916
                    for _ in range(3)]
6917
        code.put_add_traceback(self.function_name)
William Stein's avatar
William Stein committed
6918
        # We always have to fetch the exception value even if
6919
        # there is no target, because this also normalises the
William Stein's avatar
William Stein committed
6920
        # exception and stores it in the thread state.
6921 6922
        code.globalstate.use_utility_code(get_exception_utility_code)
        exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
6923 6924
        code.putln("if (__Pyx_GetException(%s) < 0) %s" % (
            exc_args, code.error_goto(self.pos)))
6925
        for x in exc_vars:
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
6926
            code.put_gotref(x)
William Stein's avatar
William Stein committed
6927
        if self.target:
6928
            self.exc_value.set_var(exc_vars[1])
6929
            self.exc_value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
6930
            self.target.generate_assignment_code(self.exc_value, code)
6931
        if self.excinfo_target is not None:
6932
            for tempvar, node in zip(exc_vars, self.excinfo_target.args):
6933
                node.set_var(tempvar)
6934

6935 6936 6937 6938
        old_break_label, old_continue_label = code.break_label, code.continue_label
        code.break_label = code.new_label('except_break')
        code.continue_label = code.new_label('except_continue')

6939
        old_exc_vars = code.funcstate.exc_vars
6940
        code.funcstate.exc_vars = exc_vars
William Stein's avatar
William Stein committed
6941
        self.body.generate_execution_code(code)
6942
        code.funcstate.exc_vars = old_exc_vars
6943 6944 6945 6946
        if not self.body.is_terminator:
            for var in exc_vars:
                code.put_decref_clear(var, py_object_type)
            code.put_goto(end_label)
6947

Stefan Behnel's avatar
Stefan Behnel committed
6948 6949 6950 6951 6952 6953 6954
        for new_label, old_label in [(code.break_label, old_break_label),
                                     (code.continue_label, old_continue_label)]:
            if code.label_used(new_label):
                code.put_label(new_label)
                for var in exc_vars:
                    code.put_decref_clear(var, py_object_type)
                code.put_goto(old_label)
Robert Bradshaw's avatar
Robert Bradshaw committed
6955
        code.break_label = old_break_label
6956
        code.continue_label = old_continue_label
6957

6958 6959
        for temp in exc_vars:
            code.funcstate.release_temp(temp)
6960

William Stein's avatar
William Stein committed
6961 6962 6963
        code.putln(
            "}")

6964 6965 6966 6967
    def generate_function_definitions(self, env, code):
        if self.target is not None:
            self.target.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
6968

6969
    def annotate(self, code):
6970
        if self.pattern:
6971 6972
            for pattern in self.pattern:
                pattern.annotate(code)
6973 6974 6975 6976
        if self.target:
            self.target.annotate(code)
        self.body.annotate(code)

William Stein's avatar
William Stein committed
6977 6978 6979 6980 6981 6982

class TryFinallyStatNode(StatNode):
    #  try ... finally statement
    #
    #  body             StatNode
    #  finally_clause   StatNode
6983
    #  finally_except_clause  deep-copy of finally_clause for exception case
6984
    #
Stefan Behnel's avatar
Stefan Behnel committed
6985 6986
    #  Each of the continue, break, return and error gotos runs
    #  into its own deep-copy of the finally block code.
William Stein's avatar
William Stein committed
6987 6988 6989
    #  In addition, if we're doing an error, we save the
    #  exception on entry to the finally block and restore
    #  it on exit.
6990

6991
    child_attrs = ["body", "finally_clause", "finally_except_clause"]
6992

6993
    preserve_exception = 1
6994

6995 6996
    # handle exception case, in addition to return/break/continue
    handle_error_case = True
6997
    func_return_type = None
6998
    finally_except_clause = None
6999

7000 7001
    is_try_finally_in_nogil = False

7002
    @staticmethod
7003 7004 7005
    def create_analysed(pos, env, body, finally_clause):
        node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
        return node
7006

William Stein's avatar
William Stein committed
7007 7008
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
7009 7010
        self.finally_except_clause = copy.deepcopy(self.finally_clause)
        self.finally_except_clause.analyse_declarations(env)
William Stein's avatar
William Stein committed
7011
        self.finally_clause.analyse_declarations(env)
7012

William Stein's avatar
William Stein committed
7013
    def analyse_expressions(self, env):
7014 7015
        self.body = self.body.analyse_expressions(env)
        self.finally_clause = self.finally_clause.analyse_expressions(env)
7016
        self.finally_except_clause = self.finally_except_clause.analyse_expressions(env)
Stefan Behnel's avatar
Stefan Behnel committed
7017 7018
        if env.return_type and not env.return_type.is_void:
            self.func_return_type = env.return_type
7019
        return self
7020

7021
    nogil_check = Node.gil_error
7022 7023
    gil_message = "Try-finally statement"

William Stein's avatar
William Stein committed
7024
    def generate_execution_code(self, code):
7025
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
7026 7027 7028 7029
        old_error_label = code.error_label
        old_labels = code.all_new_labels()
        new_labels = code.get_all_labels()
        new_error_label = code.error_label
7030 7031
        if not self.handle_error_case:
            code.error_label = old_error_label
William Stein's avatar
William Stein committed
7032
        catch_label = code.new_label()
7033 7034

        code.putln("/*try:*/ {")
Stefan Behnel's avatar
Stefan Behnel committed
7035 7036
        was_in_try_finally = code.funcstate.in_try_finally
        code.funcstate.in_try_finally = 1
7037

William Stein's avatar
William Stein committed
7038
        self.body.generate_execution_code(code)
7039

Stefan Behnel's avatar
Stefan Behnel committed
7040
        code.funcstate.in_try_finally = was_in_try_finally
7041
        code.putln("}")
7042
        code.set_all_labels(old_labels)
7043

7044
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
7045
        code.mark_pos(self.finally_clause.pos)
7046 7047
        code.putln("/*finally:*/ {")

7048 7049 7050 7051 7052 7053 7054 7055 7056
        def fresh_finally_clause(_next=[self.finally_clause]):
            # generate the original subtree once and always keep a fresh copy
            node = _next[0]
            node_copy = copy.deepcopy(node)
            if node is self.finally_clause:
                _next[0] = node_copy
            else:
                node = node_copy
            return node
7057 7058

        preserve_error = self.preserve_exception and code.label_used(new_error_label)
7059
        needs_success_cleanup = not self.finally_clause.is_terminator
7060 7061 7062

        if not self.body.is_terminator:
            code.putln('/*normal exit:*/{')
7063 7064
            fresh_finally_clause().generate_execution_code(code)
            if not self.finally_clause.is_terminator:
7065
                code.put_goto(catch_label)
7066
            code.putln('}')
7067

7068 7069
        if preserve_error:
            code.putln('/*exception exit:*/{')
7070
            code.putln("__Pyx_PyThreadState_declare")
7071 7072
            if self.is_try_finally_in_nogil:
                code.declare_gilstate()
7073 7074 7075 7076 7077 7078 7079 7080 7081 7082 7083
            if needs_success_cleanup:
                exc_lineno_cnames = tuple([
                    code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
                    for _ in range(2)])
                exc_filename_cname = code.funcstate.allocate_temp(
                    PyrexTypes.CPtrType(PyrexTypes.c_const_type(PyrexTypes.c_char_type)),
                    manage_ref=False)
            else:
                exc_lineno_cnames = exc_filename_cname = None
            exc_vars = tuple([
                code.funcstate.allocate_temp(py_object_type, manage_ref=False)
7084
                for _ in range(6)])
7085
            code.put_label(new_error_label)
7086
            self.put_error_catcher(
7087
                code, temps_to_clean_up, exc_vars, exc_lineno_cnames, exc_filename_cname)
7088
            finally_old_labels = code.all_new_labels()
7089

7090
            code.putln('{')
7091 7092
            old_exc_vars = code.funcstate.exc_vars
            code.funcstate.exc_vars = exc_vars[:3]
7093
            self.finally_except_clause.generate_execution_code(code)
7094
            code.funcstate.exc_vars = old_exc_vars
7095 7096
            code.putln('}')

7097
            if needs_success_cleanup:
7098 7099 7100 7101 7102 7103
                self.put_error_uncatcher(code, exc_vars, exc_lineno_cnames, exc_filename_cname)
                if exc_lineno_cnames:
                    for cname in exc_lineno_cnames:
                        code.funcstate.release_temp(cname)
                if exc_filename_cname:
                    code.funcstate.release_temp(exc_filename_cname)
7104
                code.put_goto(old_error_label)
7105

7106 7107 7108 7109
            for new_label, old_label in zip(code.get_all_labels(), finally_old_labels):
                if not code.label_used(new_label):
                    continue
                code.put_label(new_label)
7110
                self.put_error_cleaner(code, exc_vars)
7111
                code.put_goto(old_label)
7112 7113 7114

            for cname in exc_vars:
                code.funcstate.release_temp(cname)
7115
            code.putln('}')
7116

7117
        code.set_all_labels(old_labels)
7118
        return_label = code.return_label
7119 7120 7121 7122 7123
        for i, (new_label, old_label) in enumerate(zip(new_labels, old_labels)):
            if not code.label_used(new_label):
                continue
            if new_label == new_error_label and preserve_error:
                continue  # handled above
7124

7125 7126
            code.put('%s: ' % new_label)
            code.putln('{')
7127 7128 7129
            ret_temp = None
            if old_label == return_label and not self.finally_clause.is_terminator:
                # store away return value for later reuse
7130 7131 7132
                if (self.func_return_type and
                        not self.is_try_finally_in_nogil and
                        not isinstance(self.finally_clause, GILExitNode)):
7133 7134 7135 7136 7137
                    ret_temp = code.funcstate.allocate_temp(
                        self.func_return_type, manage_ref=False)
                    code.putln("%s = %s;" % (ret_temp, Naming.retval_cname))
                    if self.func_return_type.is_pyobject:
                        code.putln("%s = 0;" % Naming.retval_cname)
7138
            fresh_finally_clause().generate_execution_code(code)
7139 7140 7141 7142 7143 7144
            if ret_temp:
                code.putln("%s = %s;" % (Naming.retval_cname, ret_temp))
                if self.func_return_type.is_pyobject:
                    code.putln("%s = 0;" % ret_temp)
                code.funcstate.release_temp(ret_temp)
                ret_temp = None
7145
            if not self.finally_clause.is_terminator:
7146 7147
                code.put_goto(old_label)
            code.putln('}')
7148 7149

        # End finally
7150
        code.put_label(catch_label)
William Stein's avatar
William Stein committed
7151 7152 7153
        code.putln(
            "}")

7154 7155 7156 7157
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        self.finally_clause.generate_function_definitions(env, code)

7158 7159
    def put_error_catcher(self, code, temps_to_clean_up, exc_vars,
                          exc_lineno_cnames, exc_filename_cname):
7160
        code.globalstate.use_utility_code(restore_exception_utility_code)
7161 7162
        code.globalstate.use_utility_code(get_exception_utility_code)
        code.globalstate.use_utility_code(swap_exception_utility_code)
7163

7164
        code.putln(' '.join(["%s = 0;"]*len(exc_vars)) % exc_vars)
7165 7166
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7167
        code.putln("__Pyx_PyThreadState_assign")
7168

7169 7170
        for temp_name, type in temps_to_clean_up:
            code.put_xdecref_clear(temp_name, type)
7171

7172 7173 7174 7175 7176 7177 7178 7179 7180
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3)"
                   " __Pyx_ExceptionSwap(&%s, &%s, &%s);" % exc_vars[3:])
        code.putln("if ((PY_MAJOR_VERSION < 3) ||"
                   # if __Pyx_GetException() fails in Py3,
                   # store the newly raised exception instead
                   " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) "
                   "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2))
7181 7182 7183 7184 7185 7186 7187
        for var in exc_vars:
            code.put_xgotref(var)
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                exc_lineno_cnames[0], Naming.lineno_cname,
                exc_lineno_cnames[1], Naming.clineno_cname,
                exc_filename_cname, Naming.filename_cname))
7188 7189 7190 7191

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

7192
    def put_error_uncatcher(self, code, exc_vars, exc_lineno_cnames, exc_filename_cname):
7193
        code.globalstate.use_utility_code(restore_exception_utility_code)
7194
        code.globalstate.use_utility_code(reset_exception_utility_code)
7195 7196 7197

        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7198
        code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded
7199

7200 7201 7202
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
7203 7204 7205
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7206
        code.putln("}")
7207
        for var in exc_vars[:3]:
7208
            code.put_xgiveref(var)
7209
        code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3])
7210 7211 7212 7213

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

7214
        code.putln(' '.join(["%s = 0;"]*len(exc_vars)) % exc_vars)
7215 7216 7217 7218 7219
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                Naming.lineno_cname, exc_lineno_cnames[0],
                Naming.clineno_cname, exc_lineno_cnames[1],
                Naming.filename_cname, exc_filename_cname))
7220

7221
    def put_error_cleaner(self, code, exc_vars):
7222
        code.globalstate.use_utility_code(reset_exception_utility_code)
7223 7224
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
7225 7226
        code.putln("__Pyx_PyThreadState_assign")  # re-assign in case a generator yielded

7227 7228 7229
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
7230 7231 7232
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7233
        code.putln("}")
7234
        for var in exc_vars[:3]:
7235
            code.put_xdecref_clear(var, py_object_type)
7236 7237
        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()
7238
        code.putln(' '.join(["%s = 0;"]*3) % exc_vars[3:])
William Stein's avatar
William Stein committed
7239

7240 7241 7242 7243
    def annotate(self, code):
        self.body.annotate(code)
        self.finally_clause.annotate(code)

William Stein's avatar
William Stein committed
7244

7245 7246 7247 7248 7249 7250 7251 7252 7253 7254
class NogilTryFinallyStatNode(TryFinallyStatNode):
    """
    A try/finally statement that may be used in nogil code sections.
    """

    preserve_exception = False
    nogil_check = None


class GILStatNode(NogilTryFinallyStatNode):
7255 7256 7257
    #  'with gil' or 'with nogil' statement
    #
    #   state   string   'gil' or 'nogil'
7258

7259 7260
    state_temp = None

7261 7262
    def __init__(self, pos, state, body):
        self.state = state
7263
        self.create_state_temp_if_needed(pos, state, body)
7264 7265
        TryFinallyStatNode.__init__(
            self, pos,
7266 7267 7268 7269 7270
            body=body,
            finally_clause=GILExitNode(
                pos, state=state, state_temp=self.state_temp))

    def create_state_temp_if_needed(self, pos, state, body):
7271
        from .ParseTreeTransforms import YieldNodeCollector
7272 7273
        collector = YieldNodeCollector()
        collector.visitchildren(body)
7274
        if not collector.yields and not collector.awaits:
7275 7276
            return

7277 7278 7279 7280
        if state == 'gil':
            temp_type = PyrexTypes.c_gilstate_type
        else:
            temp_type = PyrexTypes.c_threadstate_ptr_type
7281
        from . import ExprNodes
7282
        self.state_temp = ExprNodes.TempNode(pos, temp_type)
7283

7284 7285 7286 7287
    def analyse_declarations(self, env):
        env._in_with_gil_block = (self.state == 'gil')
        if self.state == 'gil':
            env.has_with_gil_block = True
7288

7289 7290
        return super(GILStatNode, self).analyse_declarations(env)

7291
    def analyse_expressions(self, env):
7292 7293
        env.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
7294
        was_nogil = env.nogil
7295
        env.nogil = self.state == 'nogil'
7296
        node = TryFinallyStatNode.analyse_expressions(self, env)
7297
        env.nogil = was_nogil
7298
        return node
7299

7300
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
7301
        code.mark_pos(self.pos)
7302
        code.begin_block()
7303 7304 7305 7306 7307
        if self.state_temp:
            self.state_temp.allocate(code)
            variable = self.state_temp.result()
        else:
            variable = None
7308

7309
        old_gil_config = code.funcstate.gil_owned
7310
        if self.state == 'gil':
7311
            code.put_ensure_gil(variable=variable)
7312
            code.funcstate.gil_owned = True
7313
        else:
7314
            code.put_release_gil(variable=variable)
7315
            code.funcstate.gil_owned = False
7316

7317
        TryFinallyStatNode.generate_execution_code(self, code)
7318

7319 7320
        if self.state_temp:
            self.state_temp.release(code)
7321

7322
        code.funcstate.gil_owned = old_gil_config
7323
        code.end_block()
7324 7325 7326


class GILExitNode(StatNode):
7327 7328 7329 7330 7331
    """
    Used as the 'finally' block in a GILStatNode

    state   string   'gil' or 'nogil'
    """
7332

7333
    child_attrs = []
7334
    state_temp = None
7335

7336
    def analyse_expressions(self, env):
7337
        return self
7338 7339

    def generate_execution_code(self, code):
7340 7341 7342 7343 7344
        if self.state_temp:
            variable = self.state_temp.result()
        else:
            variable = None

7345
        if self.state == 'gil':
7346
            code.put_release_ensured_gil(variable)
7347
        else:
7348
            code.put_acquire_gil(variable)
7349 7350


7351 7352 7353 7354 7355 7356 7357
class EnsureGILNode(GILExitNode):
    """
    Ensure the GIL in nogil functions for cleanup before returning.
    """

    def generate_execution_code(self, code):
        code.put_ensure_gil(declare_gilstate=False)
7358

7359

7360 7361 7362 7363 7364
def cython_view_utility_code():
    from . import MemoryView
    return MemoryView.view_utility_code


7365 7366 7367
utility_code_for_cimports = {
    # utility code (or inlining c) in a pxd (or pyx) file.
    # TODO: Consider a generic user-level mechanism for importing
7368 7369 7370
    'cpython.array'         : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
    'cpython.array.array'   : lambda : UtilityCode.load_cached("ArrayAPI", "arrayarray.h"),
    'cython.view'           : cython_view_utility_code,
7371
}
7372

7373 7374 7375
utility_code_for_imports = {
    # utility code used when special modules are imported.
    # TODO: Consider a generic user-level mechanism for importing
7376 7377
    'asyncio': ("__Pyx_patch_asyncio", "PatchAsyncIO", "Coroutine.c"),
    'inspect': ("__Pyx_patch_inspect", "PatchInspect", "Coroutine.c"),
7378 7379 7380
}


William Stein's avatar
William Stein committed
7381 7382 7383 7384 7385
class CImportStatNode(StatNode):
    #  cimport statement
    #
    #  module_name   string           Qualified name of module being imported
    #  as_name       string or None   Name specified in "as" clause, if any
7386
    #  is_absolute   bool             True for absolute imports, False otherwise
7387 7388

    child_attrs = []
7389
    is_absolute = False
7390

William Stein's avatar
William Stein committed
7391
    def analyse_declarations(self, env):
7392 7393 7394
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
7395 7396
        module_scope = env.find_module(
            self.module_name, self.pos, relative_level=0 if self.is_absolute else -1)
William Stein's avatar
William Stein committed
7397
        if "." in self.module_name:
7398
            names = [EncodedString(name) for name in self.module_name.split(".")]
William Stein's avatar
William Stein committed
7399 7400 7401 7402 7403 7404 7405 7406 7407 7408
            top_name = names[0]
            top_module_scope = env.context.find_submodule(top_name)
            module_scope = top_module_scope
            for name in names[1:]:
                submodule_scope = module_scope.find_submodule(name)
                module_scope.declare_module(name, submodule_scope, self.pos)
                module_scope = submodule_scope
            if self.as_name:
                env.declare_module(self.as_name, module_scope, self.pos)
            else:
7409
                env.add_imported_module(module_scope)
William Stein's avatar
William Stein committed
7410 7411 7412 7413
                env.declare_module(top_name, top_module_scope, self.pos)
        else:
            name = self.as_name or self.module_name
            env.declare_module(name, module_scope, self.pos)
7414
        if self.module_name in utility_code_for_cimports:
7415
            env.use_utility_code(utility_code_for_cimports[self.module_name]())
William Stein's avatar
William Stein committed
7416 7417

    def analyse_expressions(self, env):
7418
        return self
7419

William Stein's avatar
William Stein committed
7420 7421
    def generate_execution_code(self, code):
        pass
7422

William Stein's avatar
William Stein committed
7423 7424 7425 7426

class FromCImportStatNode(StatNode):
    #  from ... cimport statement
    #
7427
    #  module_name     string                        Qualified name of module
7428
    #  relative_level  int or None                   Relative import: number of dots before module_name
7429
    #  imported_names  [(pos, name, as_name, kind)]  Names to be imported
7430

7431
    child_attrs = []
7432 7433 7434
    module_name = None
    relative_level = None
    imported_names = None
7435

William Stein's avatar
William Stein committed
7436
    def analyse_declarations(self, env):
7437 7438 7439
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
7440 7441
        if self.relative_level and self.relative_level > env.qualified_name.count('.'):
            error(self.pos, "relative cimport beyond main package is not allowed")
7442
            return
7443 7444
        module_scope = env.find_module(self.module_name, self.pos, relative_level=self.relative_level)
        module_name = module_scope.qualified_name
William Stein's avatar
William Stein committed
7445
        env.add_imported_module(module_scope)
7446
        for pos, name, as_name, kind in self.imported_names:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7447
            if name == "*":
7448
                for local_name, entry in list(module_scope.entries.items()):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7449 7450
                    env.add_imported_entry(local_name, entry, pos)
            else:
7451 7452 7453 7454
                entry = module_scope.lookup(name)
                if entry:
                    if kind and not self.declaration_matches(entry, kind):
                        entry.redeclared(pos)
7455
                    entry.used = 1
7456 7457
                else:
                    if kind == 'struct' or kind == 'union':
7458 7459
                        entry = module_scope.declare_struct_or_union(
                            name, kind=kind, scope=None, typedef_flag=0, pos=pos)
7460
                    elif kind == 'class':
7461
                        entry = module_scope.declare_c_class(name, pos=pos, module_name=module_name)
7462
                    else:
7463 7464
                        submodule_scope = env.context.find_module(
                            name, relative_to=module_scope, pos=self.pos, absolute_fallback=False)
7465 7466 7467
                        if submodule_scope.parent_module is module_scope:
                            env.declare_module(as_name or name, submodule_scope, self.pos)
                        else:
7468
                            error(pos, "Name '%s' not declared in module '%s'" % (name, module_name))
7469

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7470 7471 7472
                if entry:
                    local_name = as_name or name
                    env.add_imported_entry(local_name, entry, pos)
7473

7474
        if module_name.startswith('cpython') or module_name.startswith('cython'): # enough for now
7475
            if module_name in utility_code_for_cimports:
7476
                env.use_utility_code(utility_code_for_cimports[module_name]())
7477
            for _, name, _, _ in self.imported_names:
7478
                fqname = '%s.%s' % (module_name, name)
7479
                if fqname in utility_code_for_cimports:
7480
                    env.use_utility_code(utility_code_for_cimports[fqname]())
7481

7482
    def declaration_matches(self, entry, kind):
7483 7484 7485 7486 7487 7488 7489 7490 7491 7492 7493 7494
        if not entry.is_type:
            return 0
        type = entry.type
        if kind == 'class':
            if not type.is_extension_type:
                return 0
        else:
            if not type.is_struct_or_union:
                return 0
            if kind != type.kind:
                return 0
        return 1
William Stein's avatar
William Stein committed
7495 7496

    def analyse_expressions(self, env):
7497
        return self
7498

William Stein's avatar
William Stein committed
7499 7500 7501 7502 7503 7504 7505 7506 7507
    def generate_execution_code(self, code):
        pass


class FromImportStatNode(StatNode):
    #  from ... import statement
    #
    #  module           ImportNode
    #  items            [(string, NameNode)]
7508
    #  interned_items   [(string, NameNode, ExprNode)]
William Stein's avatar
William Stein committed
7509
    #  item             PyTempNode            used internally
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7510
    #  import_star      boolean               used internally
7511 7512

    child_attrs = ["module"]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7513
    import_star = 0
7514

William Stein's avatar
William Stein committed
7515
    def analyse_declarations(self, env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7516 7517 7518 7519 7520 7521 7522 7523 7524
        for name, target in self.items:
            if name == "*":
                if not env.is_module_scope:
                    error(self.pos, "import * only allowed at module level")
                    return
                env.has_import_star = 1
                self.import_star = 1
            else:
                target.analyse_target_declaration(env)
7525

William Stein's avatar
William Stein committed
7526
    def analyse_expressions(self, env):
7527
        from . import ExprNodes
7528
        self.module = self.module.analyse_expressions(env)
7529
        self.item = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
William Stein's avatar
William Stein committed
7530 7531
        self.interned_items = []
        for name, target in self.items:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7532 7533 7534
            if name == '*':
                for _, entry in env.entries.items():
                    if not entry.is_type and entry.type.is_extension_type:
7535
                        env.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7536 7537
                        break
            else:
7538
                entry = env.lookup(target.name)
7539 7540
                # check whether or not entry is already cimported
                if (entry.is_type and entry.type.name == name
Stefan Behnel's avatar
Stefan Behnel committed
7541
                        and hasattr(entry.type, 'module_name')):
7542 7543 7544 7545 7546
                    if entry.type.module_name == self.module.module_name.value:
                        # cimported with absolute name
                        continue
                    try:
                        # cimported with relative name
7547 7548
                        module = env.find_module(self.module.module_name.value, pos=self.pos,
                                                 relative_level=self.module.level)
7549 7550 7551
                        if entry.type.module_name == module.qualified_name:
                            continue
                    except AttributeError:
7552
                        pass
7553
                target = target.analyse_target_expression(env, None)  # FIXME?
7554 7555 7556 7557
                if target.type is py_object_type:
                    coerced_item = None
                else:
                    coerced_item = self.item.coerce_to(target.type, env)
7558
                self.interned_items.append((name, target, coerced_item))
7559
        return self
7560

William Stein's avatar
William Stein committed
7561
    def generate_execution_code(self, code):
7562
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
7563
        self.module.generate_evaluation_code(code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7564 7565 7566 7567 7568 7569
        if self.import_star:
            code.putln(
                'if (%s(%s) < 0) %s;' % (
                    Naming.import_star,
                    self.module.py_result(),
                    code.error_goto(self.pos)))
7570 7571
        item_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.item.set_cname(item_temp)
7572 7573 7574
        if self.interned_items:
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ImportFrom", "ImportExport.c"))
7575
        for name, target, coerced_item in self.interned_items:
7576
            code.putln(
7577
                '%s = __Pyx_ImportFrom(%s, %s); %s' % (
7578
                    item_temp,
7579
                    self.module.py_result(),
7580 7581
                    code.intern_identifier(name),
                    code.error_goto_if_null(item_temp, self.pos)))
7582
            code.put_gotref(item_temp)
7583 7584 7585 7586 7587 7588
            if coerced_item is None:
                target.generate_assignment_code(self.item, code)
            else:
                coerced_item.allocate_temp_result(code)
                coerced_item.generate_result_code(code)
                target.generate_assignment_code(coerced_item, code)
7589 7590
            code.put_decref_clear(item_temp, py_object_type)
        code.funcstate.release_temp(item_temp)
William Stein's avatar
William Stein committed
7591
        self.module.generate_disposal_code(code)
7592
        self.module.free_temps(code)
William Stein's avatar
William Stein committed
7593

7594

Mark Florisson's avatar
Mark Florisson committed
7595 7596 7597 7598 7599 7600 7601 7602 7603 7604
class ParallelNode(Node):
    """
    Base class for cython.parallel constructs.
    """

    nogil_check = None


class ParallelStatNode(StatNode, ParallelNode):
    """
7605
    Base class for 'with cython.parallel.parallel():' and 'for i in prange():'.
Mark Florisson's avatar
Mark Florisson committed
7606 7607 7608 7609 7610

    assignments     { Entry(var) : (var.pos, inplace_operator_or_None) }
                    assignments to variables in this parallel section

    parent          parent ParallelStatNode or None
7611 7612 7613
    is_parallel     indicates whether this node is OpenMP parallel
                    (true for #pragma omp parallel for and
                              #pragma omp parallel)
Mark Florisson's avatar
Mark Florisson committed
7614 7615 7616 7617 7618 7619 7620 7621 7622 7623 7624

    is_parallel is true for:

        #pragma omp parallel
        #pragma omp parallel for

    sections, but NOT for

        #pragma omp for

    We need this to determine the sharing attributes.
7625 7626 7627

    privatization_insertion_point   a code insertion point used to make temps
                                    private (esp. the "nsteps" temp)
7628 7629 7630 7631

    args         tuple          the arguments passed to the parallel construct
    kwargs       DictNode       the keyword arguments passed to the parallel
                                construct (replaced by its compile time value)
Mark Florisson's avatar
Mark Florisson committed
7632 7633
    """

7634
    child_attrs = ['body', 'num_threads']
Mark Florisson's avatar
Mark Florisson committed
7635 7636 7637 7638

    body = None

    is_prange = False
7639
    is_nested_prange = False
Mark Florisson's avatar
Mark Florisson committed
7640

7641
    error_label_used = False
7642

7643
    num_threads = None
7644
    chunksize = None
7645

7646 7647 7648 7649 7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662
    parallel_exc = (
        Naming.parallel_exc_type,
        Naming.parallel_exc_value,
        Naming.parallel_exc_tb,
    )

    parallel_pos_info = (
        Naming.parallel_filename,
        Naming.parallel_lineno,
        Naming.parallel_clineno,
    )

    pos_info = (
        Naming.filename_cname,
        Naming.lineno_cname,
        Naming.clineno_cname,
    )
7663

7664 7665
    critical_section_counter = 0

Mark Florisson's avatar
Mark Florisson committed
7666 7667
    def __init__(self, pos, **kwargs):
        super(ParallelStatNode, self).__init__(pos, **kwargs)
7668 7669

        # All assignments in this scope
Mark Florisson's avatar
Mark Florisson committed
7670 7671
        self.assignments = kwargs.get('assignments') or {}

7672 7673 7674 7675
        # All seen closure cnames and their temporary cnames
        self.seen_closure_vars = set()

        # Dict of variables that should be declared (first|last|)private or
7676 7677
        # reduction { Entry: (op, lastprivate) }.
        # If op is not None, it's a reduction.
7678
        self.privates = {}
Mark Florisson's avatar
Mark Florisson committed
7679

Mark Florisson's avatar
Mark Florisson committed
7680 7681 7682
        # [NameNode]
        self.assigned_nodes = []

Mark Florisson's avatar
Mark Florisson committed
7683 7684 7685
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)

7686 7687
        self.num_threads = None

7688
        if self.kwargs:
7689 7690 7691
            # Try to find num_threads and chunksize keyword arguments
            pairs = []
            for dictitem in self.kwargs.key_value_pairs:
7692 7693
                if dictitem.key.value == 'num_threads':
                    self.num_threads = dictitem.value
7694 7695 7696 7697 7698 7699
                elif self.is_prange and dictitem.key.value == 'chunksize':
                    self.chunksize = dictitem.value
                else:
                    pairs.append(dictitem)

            self.kwargs.key_value_pairs = pairs
7700

7701 7702
            try:
                self.kwargs = self.kwargs.compile_time_value(env)
7703
            except Exception as e:
7704 7705
                error(self.kwargs.pos, "Only compile-time values may be "
                                       "supplied as keyword arguments")
7706 7707 7708
        else:
            self.kwargs = {}

7709
        for kw, val in self.kwargs.items():
7710 7711 7712 7713 7714
            if kw not in self.valid_keyword_arguments:
                error(self.pos, "Invalid keyword argument: %s" % kw)
            else:
                setattr(self, kw, val)

7715
    def analyse_expressions(self, env):
7716
        if self.num_threads:
7717
            self.num_threads = self.num_threads.analyse_expressions(env)
7718 7719

        if self.chunksize:
7720
            self.chunksize = self.chunksize.analyse_expressions(env)
7721

7722
        self.body = self.body.analyse_expressions(env)
7723
        self.analyse_sharing_attributes(env)
7724

7725
        if self.num_threads is not None:
7726 7727
            if self.parent and self.parent.num_threads is not None and not self.parent.is_prange:
                error(self.pos, "num_threads already declared in outer section")
7728
            elif self.parent and not self.parent.is_prange:
7729
                error(self.pos, "num_threads must be declared in the parent parallel section")
7730
            elif (self.num_threads.type.is_int and
7731 7732 7733
                    self.num_threads.is_literal and
                    self.num_threads.compile_time_value(env) <= 0):
                error(self.pos, "argument to num_threads must be greater than 0")
7734

Mark Florisson's avatar
Mark Florisson committed
7735 7736
            if not self.num_threads.is_simple():
                self.num_threads = self.num_threads.coerce_to(
7737 7738
                    PyrexTypes.c_int_type, env).coerce_to_temp(env)
        return self
7739 7740

    def analyse_sharing_attributes(self, env):
Mark Florisson's avatar
Mark Florisson committed
7741
        """
7742 7743 7744
        Analyse the privates for this block and set them in self.privates.
        This should be called in a post-order fashion during the
        analyse_expressions phase
Mark Florisson's avatar
Mark Florisson committed
7745
        """
7746
        for entry, (pos, op) in self.assignments.items():
Mark Florisson's avatar
Mark Florisson committed
7747

7748 7749 7750 7751 7752
            if self.is_prange and not self.is_parallel:
                # closely nested prange in a with parallel block, disallow
                # assigning to privates in the with parallel block (we
                # consider it too implicit and magicky for users)
                if entry in self.parent.assignments:
7753
                    error(pos, "Cannot assign to private of outer parallel block")
7754 7755 7756 7757 7758 7759 7760
                    continue

            if not self.is_prange and op:
                # Again possible, but considered to magicky
                error(pos, "Reductions not allowed for parallel blocks")
                continue

Mark Florisson's avatar
Mark Florisson committed
7761 7762 7763
            # By default all variables should have the same values as if
            # executed sequentially
            lastprivate = True
7764
            self.propagate_var_privatization(entry, pos, op, lastprivate)
7765

7766
    def propagate_var_privatization(self, entry, pos, op, lastprivate):
7767 7768 7769 7770 7771 7772 7773 7774 7775 7776 7777 7778 7779 7780 7781 7782 7783 7784 7785 7786 7787 7788 7789 7790 7791 7792 7793 7794
        """
        Propagate the sharing attributes of a variable. If the privatization is
        determined by a parent scope, done propagate further.

        If we are a prange, we propagate our sharing attributes outwards to
        other pranges. If we are a prange in parallel block and the parallel
        block does not determine the variable private, we propagate to the
        parent of the parent. Recursion stops at parallel blocks, as they have
        no concept of lastprivate or reduction.

        So the following cases propagate:

            sum is a reduction for all loops:

                for i in prange(n):
                    for j in prange(n):
                        for k in prange(n):
                            sum += i * j * k

            sum is a reduction for both loops, local_var is private to the
            parallel with block:

                for i in prange(n):
                    with parallel:
                        local_var = ... # private to the parallel
                        for j in prange(n):
                            sum += i * j

Mark Florisson's avatar
Mark Florisson committed
7795 7796
        Nested with parallel blocks are disallowed, because they wouldn't
        allow you to propagate lastprivates or reductions:
7797 7798 7799 7800

            #pragma omp parallel for lastprivate(i)
            for i in prange(n):

Mark Florisson's avatar
Mark Florisson committed
7801 7802
                sum = 0

7803 7804 7805 7806 7807 7808 7809 7810 7811 7812 7813 7814 7815 7816 7817 7818
                #pragma omp parallel private(j, sum)
                with parallel:

                    #pragma omp parallel
                    with parallel:

                        #pragma omp for lastprivate(j) reduction(+:sum)
                        for j in prange(n):
                            sum += i

                    # sum and j are well-defined here

                # sum and j are undefined here

            # sum and j are undefined here
        """
7819
        self.privates[entry] = (op, lastprivate)
7820 7821 7822 7823 7824

        if entry.type.is_memoryviewslice:
            error(pos, "Memoryview slices can only be shared in parallel sections")
            return

7825 7826 7827 7828 7829 7830
        if self.is_prange:
            if not self.is_parallel and entry not in self.parent.assignments:
                # Parent is a parallel with block
                parent = self.parent.parent
            else:
                parent = self.parent
Mark Florisson's avatar
Mark Florisson committed
7831

7832 7833 7834
            # We don't need to propagate privates, only reductions and
            # lastprivates
            if parent and (op or lastprivate):
7835
                parent.propagate_var_privatization(entry, pos, op, lastprivate)
Mark Florisson's avatar
Mark Florisson committed
7836 7837 7838 7839 7840 7841 7842 7843 7844

    def _allocate_closure_temp(self, code, entry):
        """
        Helper function that allocate a temporary for a closure variable that
        is assigned to.
        """
        if self.parent:
            return self.parent._allocate_closure_temp(code, entry)

7845 7846 7847
        if entry.cname in self.seen_closure_vars:
            return entry.cname

7848
        cname = code.funcstate.allocate_temp(entry.type, True)
7849 7850 7851 7852 7853 7854

        # Add both the actual cname and the temp cname, as the actual cname
        # will be replaced with the temp cname on the entry
        self.seen_closure_vars.add(entry.cname)
        self.seen_closure_vars.add(cname)

Mark Florisson's avatar
Mark Florisson committed
7855 7856 7857 7858
        self.modified_entries.append((entry, entry.cname))
        code.putln("%s = %s;" % (cname, entry.cname))
        entry.cname = cname

7859
    def initialize_privates_to_nan(self, code, exclude=None):
7860
        first = True
7861

7862
        for entry, (op, lastprivate) in sorted(self.privates.items()):
7863
            if not op and (not exclude or entry != exclude):
7864
                invalid_value = entry.type.invalid_value()
7865

7866
                if invalid_value:
7867 7868 7869 7870
                    if first:
                        code.putln("/* Initialize private variables to "
                                   "invalid values */")
                        first = False
7871
                    code.putln("%s = %s;" % (entry.cname,
7872
                                             entry.type.cast_code(invalid_value)))
7873

7874
    def evaluate_before_block(self, code, expr):
Mark Florisson's avatar
Mark Florisson committed
7875
        c = self.begin_of_parallel_control_block_point_after_decls
7876 7877 7878 7879 7880 7881 7882 7883 7884 7885
        # we need to set the owner to ourselves temporarily, as
        # allocate_temp may generate a comment in the middle of our pragma
        # otherwise when DebugFlags.debug_temp_code_comments is in effect
        owner = c.funcstate.owner
        c.funcstate.owner = c
        expr.generate_evaluation_code(c)
        c.funcstate.owner = owner

        return expr.result()

7886 7887 7888 7889 7890
    def put_num_threads(self, code):
        """
        Write self.num_threads if set as the num_threads OpenMP directive
        """
        if self.num_threads is not None:
7891
            code.put(" num_threads(%s)" % self.evaluate_before_block(code, self.num_threads))
7892

7893

Mark Florisson's avatar
Mark Florisson committed
7894 7895 7896 7897 7898 7899 7900 7901 7902
    def declare_closure_privates(self, code):
        """
        If a variable is in a scope object, we need to allocate a temp and
        assign the value from the temp to the variable in the scope object
        after the parallel section. This kind of copying should be done only
        in the outermost parallel section.
        """
        self.modified_entries = []

7903
        for entry in sorted(self.assignments):
Mark Florisson's avatar
Mark Florisson committed
7904
            if entry.from_closure or entry.in_closure:
7905
                self._allocate_closure_temp(code, entry)
Mark Florisson's avatar
Mark Florisson committed
7906 7907

    def release_closure_privates(self, code):
7908 7909 7910
        """
        Release any temps used for variables in scope objects. As this is the
        outermost parallel block, we don't need to delete the cnames from
7911
        self.seen_closure_vars.
7912
        """
Mark Florisson's avatar
Mark Florisson committed
7913 7914 7915 7916 7917
        for entry, original_cname in self.modified_entries:
            code.putln("%s = %s;" % (original_cname, entry.cname))
            code.funcstate.release_temp(entry.cname)
            entry.cname = original_cname

7918 7919 7920 7921 7922 7923
    def privatize_temps(self, code, exclude_temps=()):
        """
        Make any used temporaries private. Before the relevant code block
        code.start_collecting_temps() should have been called.
        """
        if self.is_parallel:
7924
            c = self.privatization_insertion_point
7925

7926
            self.temps = temps = code.funcstate.stop_collecting_temps()
7927
            privates, firstprivates = [], []
7928
            for temp, type in sorted(temps):
7929
                if type.is_pyobject or type.is_memoryviewslice:
7930 7931 7932
                    firstprivates.append(temp)
                else:
                    privates.append(temp)
7933

7934 7935 7936 7937
            if privates:
                c.put(" private(%s)" % ", ".join(privates))
            if firstprivates:
                c.put(" firstprivate(%s)" % ", ".join(firstprivates))
7938 7939 7940 7941 7942 7943 7944 7945

            if self.breaking_label_used:
                shared_vars = [Naming.parallel_why]
                if self.error_label_used:
                    shared_vars.extend(self.parallel_exc)
                    c.put(" private(%s, %s, %s)" % self.pos_info)

                c.put(" shared(%s)" % ', '.join(shared_vars))
7946

7947 7948 7949 7950
    def cleanup_temps(self, code):
        # Now clean up any memoryview slice and object temporaries
        if self.is_parallel and not self.is_nested_prange:
            code.putln("/* Clean up any temporaries */")
7951
            for temp, type in sorted(self.temps):
7952 7953 7954 7955 7956
                if type.is_memoryviewslice:
                    code.put_xdecref_memoryviewslice(temp, have_gil=False)
                elif type.is_pyobject:
                    code.put_xdecref(temp, type)
                    code.putln("%s = NULL;" % temp)
7957

7958
    def setup_parallel_control_flow_block(self, code):
7959
        """
7960 7961
        Sets up a block that surrounds the parallel block to determine
        how the parallel section was exited. Any kind of return is
7962 7963 7964
        trapped (break, continue, return, exceptions). This is the idea:

        {
7965
            int why = 0;
7966 7967 7968 7969 7970 7971 7972

            #pragma omp parallel
            {
                return # -> goto new_return_label;
                goto end_parallel;

            new_return_label:
7973
                why = 3;
7974
                goto end_parallel;
7975

7976
            end_parallel:;
7977
                #pragma omp flush(why) # we need to flush for every iteration
7978 7979
            }

7980
            if (why == 3)
7981 7982 7983 7984 7985 7986 7987 7988
                goto old_return_label;
        }
        """
        self.old_loop_labels = code.new_loop_labels()
        self.old_error_label = code.new_error_label()
        self.old_return_label = code.return_label
        code.return_label = code.new_label(name="return")

7989 7990
        code.begin_block() # parallel control flow block
        self.begin_of_parallel_control_block_point = code.insertion_point()
Mark Florisson's avatar
Mark Florisson committed
7991
        self.begin_of_parallel_control_block_point_after_decls = code.insertion_point()
7992

7993 7994
        self.undef_builtin_expect_apple_gcc_bug(code)

7995 7996 7997 7998 7999 8000 8001 8002 8003 8004 8005 8006
    def begin_parallel_block(self, code):
        """
        Each OpenMP thread in a parallel section that contains a with gil block
        must have the thread-state initialized. The call to
        PyGILState_Release() then deallocates our threadstate. If we wouldn't
        do this, each with gil block would allocate and deallocate one, thereby
        losing exception information before it can be saved before leaving the
        parallel section.
        """
        self.begin_of_parallel_block = code.insertion_point()

    def end_parallel_block(self, code):
8007 8008 8009 8010 8011 8012 8013 8014 8015
        """
        To ensure all OpenMP threads have thread states, we ensure the GIL
        in each thread (which creates a thread state if it doesn't exist),
        after which we release the GIL.
        On exit, reacquire the GIL and release the thread state.

        If compiled without OpenMP support (at the C level), then we still have
        to acquire the GIL to decref any object temporaries.
        """
8016 8017 8018 8019
        if self.error_label_used:
            begin_code = self.begin_of_parallel_block
            end_code = code

8020
            begin_code.putln("#ifdef _OPENMP")
8021 8022
            begin_code.put_ensure_gil(declare_gilstate=True)
            begin_code.putln("Py_BEGIN_ALLOW_THREADS")
8023
            begin_code.putln("#endif /* _OPENMP */")
8024

8025
            end_code.putln("#ifdef _OPENMP")
8026
            end_code.putln("Py_END_ALLOW_THREADS")
8027 8028 8029 8030 8031
            end_code.putln("#else")
            end_code.put_safe("{\n")
            end_code.put_ensure_gil()
            end_code.putln("#endif /* _OPENMP */")
            self.cleanup_temps(end_code)
8032
            end_code.put_release_ensured_gil()
8033 8034
            end_code.putln("#ifndef _OPENMP")
            end_code.put_safe("}\n")
8035
            end_code.putln("#endif /* _OPENMP */")
8036

8037 8038 8039 8040
    def trap_parallel_exit(self, code, should_flush=False):
        """
        Trap any kind of return inside a parallel construct. 'should_flush'
        indicates whether the variable should be flushed, which is needed by
8041 8042 8043 8044 8045 8046 8047 8048 8049
        prange to skip the loop. It also indicates whether we need to register
        a continue (we need this for parallel blocks, but not for prange
        loops, as it is a direct jump there).

        It uses the same mechanism as try/finally:
            1 continue
            2 break
            3 return
            4 error
8050
        """
8051
        save_lastprivates_label = code.new_label()
8052
        dont_return_label = code.new_label()
8053 8054 8055

        self.any_label_used = False
        self.breaking_label_used = False
8056
        self.error_label_used = False
8057

8058 8059 8060 8061 8062 8063
        self.parallel_private_temps = []

        all_labels = code.get_all_labels()

        # Figure this out before starting to generate any code
        for label in all_labels:
8064 8065
            if code.label_used(label):
                self.breaking_label_used = (self.breaking_label_used or
8066 8067 8068 8069 8070 8071 8072 8073 8074 8075 8076
                                            label != code.continue_label)
                self.any_label_used = True

        if self.any_label_used:
            code.put_goto(dont_return_label)

        for i, label in enumerate(all_labels):
            if not code.label_used(label):
                continue

            is_continue_label = label == code.continue_label
8077

8078
            code.put_label(label)
8079

8080 8081 8082 8083
            if not (should_flush and is_continue_label):
                if label == code.error_label:
                    self.error_label_used = True
                    self.fetch_parallel_exception(code)
8084

8085
                code.putln("%s = %d;" % (Naming.parallel_why, i + 1))
8086

8087 8088 8089 8090
            if (self.breaking_label_used and self.is_prange and not
                    is_continue_label):
                code.put_goto(save_lastprivates_label)
            else:
8091 8092
                code.put_goto(dont_return_label)

8093
        if self.any_label_used:
8094 8095 8096 8097 8098
            if self.is_prange and self.breaking_label_used:
                # Don't rely on lastprivate, save our lastprivates
                code.put_label(save_lastprivates_label)
                self.save_parallel_vars(code)

8099 8100 8101
            code.put_label(dont_return_label)

            if should_flush and self.breaking_label_used:
8102 8103
                code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_why)

8104 8105 8106 8107 8108 8109 8110 8111 8112
    def save_parallel_vars(self, code):
        """
        The following shenanigans are instated when we break, return or
        propagate errors from a prange. In this case we cannot rely on
        lastprivate() to do its job, as no iterations may have executed yet
        in the last thread, leaving the values undefined. It is most likely
        that the breaking thread has well-defined values of the lastprivate
        variables, so we keep those values.
        """
8113
        section_name = "__pyx_parallel_lastprivates%d" % self.critical_section_counter
8114 8115 8116 8117 8118 8119 8120 8121
        code.putln_openmp("#pragma omp critical(%s)" % section_name)
        ParallelStatNode.critical_section_counter += 1

        code.begin_block() # begin critical section

        c = self.begin_of_parallel_control_block_point

        temp_count = 0
8122
        for entry, (op, lastprivate) in sorted(self.privates.items()):
8123 8124 8125
            if not lastprivate or entry.type.is_pyobject:
                continue

8126
            type_decl = entry.type.empty_declaration_code()
8127 8128 8129 8130 8131
            temp_cname = "__pyx_parallel_temp%d" % temp_count
            private_cname = entry.cname

            temp_count += 1

8132 8133
            invalid_value = entry.type.invalid_value()
            if invalid_value:
8134
                init = ' = ' + entry.type.cast_code(invalid_value)
8135 8136
            else:
                init = ''
8137
            # Declare the parallel private in the outer block
8138
            c.putln("%s %s%s;" % (type_decl, temp_cname, init))
8139 8140 8141 8142 8143 8144 8145 8146

            # Initialize before escaping
            code.putln("%s = %s;" % (temp_cname, private_cname))

            self.parallel_private_temps.append((temp_cname, private_cname))

        code.end_block() # end critical section

8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163 8164 8165 8166 8167 8168 8169 8170 8171 8172 8173 8174 8175 8176 8177 8178 8179 8180
    def fetch_parallel_exception(self, code):
        """
        As each OpenMP thread may raise an exception, we need to fetch that
        exception from the threadstate and save it for after the parallel
        section where it can be re-raised in the master thread.

        Although it would seem that __pyx_filename, __pyx_lineno and
        __pyx_clineno are only assigned to under exception conditions (i.e.,
        when we have the GIL), and thus should be allowed to be shared without
        any race condition, they are in fact subject to the same race
        conditions that they were previously when they were global variables
        and functions were allowed to release the GIL:

            thread A                thread B
                acquire
                set lineno
                release
                                        acquire
                                        set lineno
                                        release
                acquire
                fetch exception
                release
                                        skip the fetch

                deallocate threadstate  deallocate threadstate
        """
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

        code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_exc_type)
        code.putln(
            "if (!%s) {" % Naming.parallel_exc_type)

8181
        code.putln("__Pyx_ErrFetchWithState(&%s, &%s, &%s);" % self.parallel_exc)
8182
        pos_info = chain(*zip(self.parallel_pos_info, self.pos_info))
8183
        code.funcstate.uses_error_indicator = True
8184
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
8185
        code.put_gotref(Naming.parallel_exc_type)
8186 8187 8188 8189 8190 8191 8192 8193 8194 8195 8196 8197

        code.putln(
            "}")

        code.put_release_ensured_gil()
        code.end_block()

    def restore_parallel_exception(self, code):
        "Re-raise a parallel exception"
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

8198
        code.put_giveref(Naming.parallel_exc_type)
8199
        code.putln("__Pyx_ErrRestoreWithState(%s, %s, %s);" % self.parallel_exc)
8200 8201 8202 8203 8204
        pos_info = chain(*zip(self.pos_info, self.parallel_pos_info))
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))

        code.put_release_ensured_gil()
        code.end_block()
8205 8206

    def restore_labels(self, code):
8207 8208 8209 8210 8211 8212
        """
        Restore all old labels. Call this before the 'else' clause to for
        loops and always before ending the parallel control flow block.
        """
        code.set_all_labels(self.old_loop_labels + (self.old_return_label,
                                                    self.old_error_label))
8213

8214 8215
    def end_parallel_control_flow_block(
            self, code, break_=False, continue_=False, return_=False):
8216 8217 8218 8219 8220
        """
        This ends the parallel control flow block and based on how the parallel
        section was exited, takes the corresponding action. The break_ and
        continue_ parameters indicate whether these should be propagated
        outwards:
8221

8222 8223 8224 8225 8226 8227 8228
            for i in prange(...):
                with cython.parallel.parallel():
                    continue

        Here break should be trapped in the parallel block, and propagated to
        the for loop.
        """
8229 8230 8231 8232
        c = self.begin_of_parallel_control_block_point

        # Firstly, always prefer errors over returning, continue or break
        if self.error_label_used:
8233 8234
            c.putln("const char *%s = NULL; int %s = 0, %s = 0;" % self.parallel_pos_info)
            c.putln("PyObject *%s = NULL, *%s = NULL, *%s = NULL;" % self.parallel_exc)
8235 8236 8237 8238 8239 8240 8241 8242 8243

            code.putln(
                "if (%s) {" % Naming.parallel_exc_type)
            code.putln("/* This may have been overridden by a continue, "
                       "break or return in another thread. Prefer the error. */")
            code.putln("%s = 4;" % Naming.parallel_why)
            code.putln(
                "}")

8244 8245 8246 8247 8248 8249 8250
        if continue_:
            any_label_used = self.any_label_used
        else:
            any_label_used = self.breaking_label_used

        if any_label_used:
            # __pyx_parallel_why is used, declare and initialize
8251 8252
            c.putln("int %s;" % Naming.parallel_why)
            c.putln("%s = 0;" % Naming.parallel_why)
8253

8254 8255 8256 8257 8258 8259
            code.putln(
                "if (%s) {" % Naming.parallel_why)

            for temp_cname, private_cname in self.parallel_private_temps:
                code.putln("%s = %s;" % (private_cname, temp_cname))

8260
            code.putln("switch (%s) {" % Naming.parallel_why)
8261 8262 8263 8264 8265 8266 8267 8268
            if continue_:
                code.put("    case 1: ")
                code.put_goto(code.continue_label)

            if break_:
                code.put("    case 2: ")
                code.put_goto(code.break_label)

8269 8270 8271
            if return_:
                code.put("    case 3: ")
                code.put_goto(code.return_label)
8272 8273

            if self.error_label_used:
8274
                code.globalstate.use_utility_code(restore_exception_utility_code)
8275 8276 8277
                code.putln("    case 4:")
                self.restore_parallel_exception(code)
                code.put_goto(code.error_label)
8278

8279 8280 8281
            code.putln("}") # end switch
            code.putln(
                "}") # end if
8282 8283

        code.end_block() # end parallel control flow block
8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295
        self.redef_builtin_expect_apple_gcc_bug(code)

    # FIXME: improve with version number for OS X Lion
    buggy_platform_macro_condition = "(defined(__APPLE__) || defined(__OSX__))"
    have_expect_condition = "(defined(__GNUC__) && " \
                             "(__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))))"
    redef_condition = "(%s && %s)" % (buggy_platform_macro_condition, have_expect_condition)

    def undef_builtin_expect_apple_gcc_bug(self, code):
        """
        A bug on OS X Lion disallows __builtin_expect macros. This code avoids them
        """
8296
        if not self.parent:
8297
            code.undef_builtin_expect(self.redef_condition)
8298 8299

    def redef_builtin_expect_apple_gcc_bug(self, code):
8300
        if not self.parent:
8301
            code.redef_builtin_expect(self.redef_condition)
8302

Mark Florisson's avatar
Mark Florisson committed
8303 8304 8305

class ParallelWithBlockNode(ParallelStatNode):
    """
8306
    This node represents a 'with cython.parallel.parallel():' block
Mark Florisson's avatar
Mark Florisson committed
8307 8308
    """

8309 8310 8311 8312 8313 8314 8315 8316 8317
    valid_keyword_arguments = ['num_threads']

    num_threads = None

    def analyse_declarations(self, env):
        super(ParallelWithBlockNode, self).analyse_declarations(env)
        if self.args:
            error(self.pos, "cython.parallel.parallel() does not take "
                            "positional arguments")
8318

Mark Florisson's avatar
Mark Florisson committed
8319 8320
    def generate_execution_code(self, code):
        self.declare_closure_privates(code)
8321
        self.setup_parallel_control_flow_block(code)
Mark Florisson's avatar
Mark Florisson committed
8322 8323 8324

        code.putln("#ifdef _OPENMP")
        code.put("#pragma omp parallel ")
8325 8326

        if self.privates:
8327
            privates = [e.cname for e in self.privates
8328
                        if not e.type.is_pyobject]
8329
            code.put('private(%s)' % ', '.join(sorted(privates)))
8330

8331
        self.privatization_insertion_point = code.insertion_point()
8332
        self.put_num_threads(code)
8333
        code.putln("")
8334

8335 8336
        code.putln("#endif /* _OPENMP */")

8337
        code.begin_block()  # parallel block
8338
        self.begin_parallel_block(code)
8339
        self.initialize_privates_to_nan(code)
8340
        code.funcstate.start_collecting_temps()
Mark Florisson's avatar
Mark Florisson committed
8341
        self.body.generate_execution_code(code)
8342
        self.trap_parallel_exit(code)
8343
        self.privatize_temps(code)
8344
        self.end_parallel_block(code)
8345
        code.end_block()  # end parallel block
8346

8347 8348
        continue_ = code.label_used(code.continue_label)
        break_ = code.label_used(code.break_label)
8349
        return_ = code.label_used(code.return_label)
Mark Florisson's avatar
Mark Florisson committed
8350

8351 8352
        self.restore_labels(code)
        self.end_parallel_control_flow_block(code, break_=break_,
8353 8354
                                             continue_=continue_,
                                             return_=return_)
Mark Florisson's avatar
Mark Florisson committed
8355 8356 8357 8358 8359 8360 8361 8362 8363 8364 8365
        self.release_closure_privates(code)


class ParallelRangeNode(ParallelStatNode):
    """
    This node represents a 'for i in cython.parallel.prange():' construct.

    target       NameNode       the target iteration variable
    else_clause  Node or None   the else clause of this loop
    """

8366 8367
    child_attrs = ['body', 'target', 'else_clause', 'args', 'num_threads',
                   'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8368 8369 8370 8371 8372 8373

    body = target = else_clause = args = None

    start = stop = step = None

    is_prange = True
8374

8375
    nogil = None
8376 8377
    schedule = None

8378
    valid_keyword_arguments = ['schedule', 'nogil', 'num_threads', 'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8379

8380 8381 8382 8383 8384
    def __init__(self, pos, **kwds):
        super(ParallelRangeNode, self).__init__(pos, **kwds)
        # Pretend to be a ForInStatNode for control flow analysis
        self.iterator = PassStatNode(pos)

Mark Florisson's avatar
Mark Florisson committed
8385 8386 8387 8388 8389 8390 8391 8392 8393 8394 8395 8396 8397 8398 8399 8400 8401
    def analyse_declarations(self, env):
        super(ParallelRangeNode, self).analyse_declarations(env)
        self.target.analyse_target_declaration(env)
        if self.else_clause is not None:
            self.else_clause.analyse_declarations(env)

        if not self.args or len(self.args) > 3:
            error(self.pos, "Invalid number of positional arguments to prange")
            return

        if len(self.args) == 1:
            self.stop, = self.args
        elif len(self.args) == 2:
            self.start, self.stop = self.args
        else:
            self.start, self.stop, self.step = self.args

Mark Florisson's avatar
Mark Florisson committed
8402 8403 8404
        if hasattr(self.schedule, 'decode'):
            self.schedule = self.schedule.decode('ascii')

8405 8406
        if self.schedule not in (None, 'static', 'dynamic', 'guided', 'runtime'):
            error(self.pos, "Invalid schedule argument to prange: %s" % (self.schedule,))
Mark Florisson's avatar
Mark Florisson committed
8407 8408

    def analyse_expressions(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
8409
        was_nogil = env.nogil
8410 8411 8412
        if self.nogil:
            env.nogil = True

8413 8414
        if self.target is None:
            error(self.pos, "prange() can only be used as part of a for loop")
8415
            return self
Mark Florisson's avatar
Mark Florisson committed
8416

8417
        self.target = self.target.analyse_target_types(env)
Mark Florisson's avatar
Mark Florisson committed
8418

8419 8420 8421 8422 8423 8424 8425 8426
        if not self.target.type.is_numeric:
            # Not a valid type, assume one for now anyway

            if not self.target.type.is_pyobject:
                # nogil_check will catch the is_pyobject case
                error(self.target.pos,
                      "Must be of numeric type, not %s" % self.target.type)

8427
            self.index_type = PyrexTypes.c_py_ssize_t_type
8428 8429
        else:
            self.index_type = self.target.type
8430 8431 8432 8433
            if not self.index_type.signed:
                warning(self.target.pos,
                        "Unsigned index type not allowed before OpenMP 3.0",
                        level=2)
Mark Florisson's avatar
Mark Florisson committed
8434 8435 8436 8437 8438 8439 8440 8441 8442

        # Setup start, stop and step, allocating temps if needed
        self.names = 'start', 'stop', 'step'
        start_stop_step = self.start, self.stop, self.step

        for node, name in zip(start_stop_step, self.names):
            if node is not None:
                node.analyse_types(env)
                if not node.type.is_numeric:
8443 8444
                    error(node.pos, "%s argument must be numeric" % name)
                    continue
Mark Florisson's avatar
Mark Florisson committed
8445 8446 8447 8448 8449 8450 8451 8452

                if not node.is_literal:
                    node = node.coerce_to_temp(env)
                    setattr(self, name, node)

                # As we range from 0 to nsteps, computing the index along the
                # way, we need a fitting type for 'i' and 'nsteps'
                self.index_type = PyrexTypes.widest_numeric_type(
8453
                    self.index_type, node.type)
Mark Florisson's avatar
Mark Florisson committed
8454 8455

        if self.else_clause is not None:
8456
            self.else_clause = self.else_clause.analyse_expressions(env)
Mark Florisson's avatar
Mark Florisson committed
8457

8458 8459 8460 8461 8462 8463 8464 8465
        # Although not actually an assignment in this scope, it should be
        # treated as such to ensure it is unpacked if a closure temp, and to
        # ensure lastprivate behaviour and propagation. If the target index is
        # not a NameNode, it won't have an entry, and an error was issued by
        # ParallelRangeTransform
        if hasattr(self.target, 'entry'):
            self.assignments[self.target.entry] = self.target.pos, None

8466
        node = super(ParallelRangeNode, self).analyse_expressions(env)
8467

8468 8469 8470
        if node.chunksize:
            if not node.schedule:
                error(node.chunksize.pos,
8471
                      "Must provide schedule with chunksize")
8472 8473
            elif node.schedule == 'runtime':
                error(node.chunksize.pos,
8474
                      "Chunksize not valid for the schedule runtime")
8475 8476 8477 8478
            elif (node.chunksize.type.is_int and
                  node.chunksize.is_literal and
                  node.chunksize.compile_time_value(env) <= 0):
                error(node.chunksize.pos, "Chunksize must not be negative")
8479

8480 8481
            node.chunksize = node.chunksize.coerce_to(
                PyrexTypes.c_int_type, env).coerce_to_temp(env)
8482

8483
        if node.nogil:
8484 8485
            env.nogil = was_nogil

8486 8487 8488
        node.is_nested_prange = node.parent and node.parent.is_prange
        if node.is_nested_prange:
            parent = node
8489 8490 8491
            while parent.parent and parent.parent.is_prange:
                parent = parent.parent

8492 8493 8494 8495
            parent.assignments.update(node.assignments)
            parent.privates.update(node.privates)
            parent.assigned_nodes.extend(node.assigned_nodes)
        return node
8496

Mark Florisson's avatar
Mark Florisson committed
8497 8498
    def nogil_check(self, env):
        names = 'start', 'stop', 'step', 'target'
8499
        nodes = self.start, self.stop, self.step, self.target
Mark Florisson's avatar
Mark Florisson committed
8500 8501 8502 8503 8504 8505 8506 8507 8508 8509 8510 8511 8512 8513 8514 8515 8516 8517 8518 8519 8520 8521 8522 8523 8524 8525 8526 8527 8528 8529 8530 8531 8532 8533 8534 8535 8536 8537 8538 8539 8540 8541 8542 8543 8544
        for name, node in zip(names, nodes):
            if node is not None and node.type.is_pyobject:
                error(node.pos, "%s may not be a Python object "
                                "as we don't have the GIL" % name)

    def generate_execution_code(self, code):
        """
        Generate code in the following steps

            1)  copy any closure variables determined thread-private
                into temporaries

            2)  allocate temps for start, stop and step

            3)  generate a loop that calculates the total number of steps,
                which then computes the target iteration variable for every step:

                    for i in prange(start, stop, step):
                        ...

                becomes

                    nsteps = (stop - start) / step;
                    i = start;

                    #pragma omp parallel for lastprivate(i)
                    for (temp = 0; temp < nsteps; temp++) {
                        i = start + step * temp;
                        ...
                    }

                Note that accumulation of 'i' would have a data dependency
                between iterations.

                Also, you can't do this

                    for (i = start; i < stop; i += step)
                        ...

                as the '<' operator should become '>' for descending loops.
                'for i from x < i < y:' does not suffer from this problem
                as the relational operator is known at compile time!

            4) release our temps and write back any private closure variables
        """
8545
        self.declare_closure_privates(code)
Mark Florisson's avatar
Mark Florisson committed
8546 8547 8548 8549 8550 8551 8552 8553

        # This can only be a NameNode
        target_index_cname = self.target.entry.cname

        # This will be used as the dict to format our code strings, holding
        # the start, stop , step, temps and target cnames
        fmt_dict = {
            'target': target_index_cname,
8554
            'target_type': self.target.type.empty_declaration_code()
Mark Florisson's avatar
Mark Florisson committed
8555 8556 8557 8558 8559 8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575 8576 8577
        }

        # Setup start, stop and step, allocating temps if needed
        start_stop_step = self.start, self.stop, self.step
        defaults = '0', '0', '1'
        for node, name, default in zip(start_stop_step, self.names, defaults):
            if node is None:
                result = default
            elif node.is_literal:
                result = node.get_constant_c_result_code()
            else:
                node.generate_evaluation_code(code)
                result = node.result()

            fmt_dict[name] = result

        fmt_dict['i'] = code.funcstate.allocate_temp(self.index_type, False)
        fmt_dict['nsteps'] = code.funcstate.allocate_temp(self.index_type, False)

        # TODO: check if the step is 0 and if so, raise an exception in a
        # 'with gil' block. For now, just abort
        code.putln("if (%(step)s == 0) abort();" % fmt_dict)

8578
        self.setup_parallel_control_flow_block(code) # parallel control flow block
8579

8580
        self.control_flow_var_code_point = code.insertion_point()
8581

8582
        # Note: nsteps is private in an outer scope if present
8583
        code.putln("%(nsteps)s = (%(stop)s - %(start)s + %(step)s - %(step)s/abs(%(step)s)) / %(step)s;" % fmt_dict)
Mark Florisson's avatar
Mark Florisson committed
8584

8585 8586 8587 8588 8589 8590 8591
        # The target iteration variable might not be initialized, do it only if
        # we are executing at least 1 iteration, otherwise we should leave the
        # target unaffected. The target iteration variable is firstprivate to
        # shut up compiler warnings caused by lastprivate, as the compiler
        # erroneously believes that nsteps may be <= 0, leaving the private
        # target index uninitialized
        code.putln("if (%(nsteps)s > 0)" % fmt_dict)
8592
        code.begin_block() # if block
Mark Florisson's avatar
Mark Florisson committed
8593
        self.generate_loop(code, fmt_dict)
8594
        code.end_block() # end if block
Mark Florisson's avatar
Mark Florisson committed
8595

8596 8597 8598
        self.restore_labels(code)

        if self.else_clause:
8599
            if self.breaking_label_used:
8600
                code.put("if (%s < 2)" % Naming.parallel_why)
8601 8602 8603 8604 8605 8606 8607

            code.begin_block() # else block
            code.putln("/* else */")
            self.else_clause.generate_execution_code(code)
            code.end_block() # end else block

        # ------ cleanup ------
8608
        self.end_parallel_control_flow_block(code) # end parallel control flow block
8609

Mark Florisson's avatar
Mark Florisson committed
8610 8611
        # And finally, release our privates and write back any closure
        # variables
8612
        for temp in start_stop_step + (self.chunksize, self.num_threads):
Mark Florisson's avatar
Mark Florisson committed
8613 8614 8615 8616 8617 8618 8619 8620 8621 8622
            if temp is not None:
                temp.generate_disposal_code(code)
                temp.free_temps(code)

        code.funcstate.release_temp(fmt_dict['i'])
        code.funcstate.release_temp(fmt_dict['nsteps'])

        self.release_closure_privates(code)

    def generate_loop(self, code, fmt_dict):
8623 8624 8625 8626
        if self.is_nested_prange:
            code.putln("#if 0")
        else:
            code.putln("#ifdef _OPENMP")
Mark Florisson's avatar
Mark Florisson committed
8627 8628 8629

        if not self.is_parallel:
            code.put("#pragma omp for")
8630
            self.privatization_insertion_point = code.insertion_point()
8631
            reduction_codepoint = self.parent.privatization_insertion_point
Mark Florisson's avatar
Mark Florisson committed
8632
        else:
8633 8634
            code.put("#pragma omp parallel")
            self.privatization_insertion_point = code.insertion_point()
8635
            reduction_codepoint = self.privatization_insertion_point
8636 8637 8638 8639 8640 8641 8642 8643
            code.putln("")
            code.putln("#endif /* _OPENMP */")

            code.begin_block() # pragma omp parallel begin block

            # Initialize the GIL if needed for this thread
            self.begin_parallel_block(code)

8644 8645 8646 8647
            if self.is_nested_prange:
                code.putln("#if 0")
            else:
                code.putln("#ifdef _OPENMP")
8648
            code.put("#pragma omp for")
Mark Florisson's avatar
Mark Florisson committed
8649

8650
        for entry, (op, lastprivate) in sorted(self.privates.items()):
Mark Florisson's avatar
Mark Florisson committed
8651
            # Don't declare the index variable as a reduction
8652
            if op and op in "+*-&^|" and entry != self.target.entry:
8653 8654 8655
                if entry.type.is_pyobject:
                    error(self.pos, "Python objects cannot be reductions")
                else:
8656 8657 8658 8659
                    #code.put(" reduction(%s:%s)" % (op, entry.cname))
                    # This is the only way reductions + nesting works in gcc4.5
                    reduction_codepoint.put(
                                " reduction(%s:%s)" % (op, entry.cname))
8660
            else:
8661 8662
                if entry == self.target.entry:
                    code.put(" firstprivate(%s)" % entry.cname)
8663 8664
                    code.put(" lastprivate(%s)" % entry.cname)
                    continue
Mark Florisson's avatar
Mark Florisson committed
8665

8666
                if not entry.type.is_pyobject:
8667 8668 8669 8670
                    if lastprivate:
                        private = 'lastprivate'
                    else:
                        private = 'private'
Mark Florisson's avatar
Mark Florisson committed
8671

8672
                    code.put(" %s(%s)" % (private, entry.cname))
8673

Mark Florisson's avatar
Mark Florisson committed
8674
        if self.schedule:
8675
            if self.chunksize:
8676
                chunksize = ", %s" % self.evaluate_before_block(code, self.chunksize)
8677 8678 8679 8680
            else:
                chunksize = ""

            code.put(" schedule(%s%s)" % (self.schedule, chunksize))
8681

8682
        self.put_num_threads(reduction_codepoint)
8683

8684 8685
        code.putln("")
        code.putln("#endif /* _OPENMP */")
Mark Florisson's avatar
Mark Florisson committed
8686 8687

        code.put("for (%(i)s = 0; %(i)s < %(nsteps)s; %(i)s++)" % fmt_dict)
8688
        code.begin_block()  # for loop block
8689

8690
        guard_around_body_codepoint = code.insertion_point()
8691

8692 8693
        # Start if guard block around the body. This may be unnecessary, but
        # at least it doesn't spoil indentation
Mark Florisson's avatar
Mark Florisson committed
8694
        code.begin_block()
8695

8696
        code.putln("%(target)s = (%(target_type)s)(%(start)s + %(step)s * %(i)s);" % fmt_dict)
8697 8698
        self.initialize_privates_to_nan(code, exclude=self.target.entry)

8699 8700 8701
        if self.is_parallel:
            code.funcstate.start_collecting_temps()

Mark Florisson's avatar
Mark Florisson committed
8702
        self.body.generate_execution_code(code)
8703
        self.trap_parallel_exit(code, should_flush=True)
8704 8705
        self.privatize_temps(code)

8706 8707 8708
        if self.breaking_label_used:
            # Put a guard around the loop body in case return, break or
            # exceptions might be used
8709
            guard_around_body_codepoint.putln("if (%s < 2)" % Naming.parallel_why)
8710

8711 8712
        code.end_block()  # end guard around loop body
        code.end_block()  # end for loop block
Mark Florisson's avatar
Mark Florisson committed
8713

8714 8715 8716
        if self.is_parallel:
            # Release the GIL and deallocate the thread state
            self.end_parallel_block(code)
8717
            code.end_block()  # pragma omp parallel end block
8718

8719

8720 8721 8722 8723 8724 8725 8726 8727
class CnameDecoratorNode(StatNode):
    """
    This node is for the cname decorator in CythonUtilityCode:

        @cname('the_cname')
        cdef func(...):
            ...

8728 8729
    In case of a cdef class the cname specifies the objstruct_cname.

8730 8731 8732 8733 8734 8735 8736 8737
    node        the node to which the cname decorator is applied
    cname       the cname the node should get
    """

    child_attrs = ['node']

    def analyse_declarations(self, env):
        self.node.analyse_declarations(env)
8738

8739 8740 8741 8742 8743
        node = self.node
        if isinstance(node, CompilerDirectivesNode):
            node = node.body.stats[0]

        self.is_function = isinstance(node, FuncDefNode)
8744
        is_struct_or_enum = isinstance(node, (CStructOrUnionDefNode, CEnumDefNode))
8745
        e = node.entry
8746 8747 8748 8749

        if self.is_function:
            e.cname = self.cname
            e.func_cname = self.cname
8750
            e.used = True
8751 8752
            if e.pyfunc_cname and '.' in e.pyfunc_cname:
                e.pyfunc_cname = self.mangle(e.pyfunc_cname)
8753 8754
        elif is_struct_or_enum:
            e.cname = e.type.cname = self.cname
8755
        else:
8756
            scope = node.scope
8757 8758

            e.cname = self.cname
8759
            e.type.objstruct_cname = self.cname + '_obj'
8760
            e.type.typeobj_cname = Naming.typeobj_prefix + self.cname
8761
            e.type.typeptr_cname = self.cname + '_type'
8762
            e.type.scope.namespace_cname = e.type.typeptr_cname
8763

8764
            e.as_variable.cname = e.type.typeptr_cname
8765 8766 8767

            scope.scope_prefix = self.cname + "_"

8768
            for name, entry in scope.entries.items():
8769
                if entry.func_cname:
8770 8771 8772 8773 8774 8775 8776 8777 8778
                    entry.func_cname = self.mangle(entry.cname)
                if entry.pyfunc_cname:
                    entry.pyfunc_cname = self.mangle(entry.pyfunc_cname)

    def mangle(self, cname):
        if '.' in cname:
            # remove __pyx_base from func_cname
            cname = cname.split('.')[-1]
        return '%s_%s' % (self.cname, cname)
8779 8780

    def analyse_expressions(self, env):
8781 8782
        self.node = self.node.analyse_expressions(env)
        return self
8783 8784

    def generate_function_definitions(self, env, code):
8785
        "Ensure a prototype for every @cname method in the right place"
8786 8787 8788 8789 8790 8791
        if self.is_function and env.is_c_class_scope:
            # method in cdef class, generate a prototype in the header
            h_code = code.globalstate['utility_code_proto']

            if isinstance(self.node, DefNode):
                self.node.generate_function_header(
8792
                    h_code, with_pymethdef=False, proto_only=True)
8793
            else:
8794
                from . import ModuleNode
8795 8796 8797 8798 8799
                entry = self.node.entry
                cname = entry.cname
                entry.cname = entry.func_cname

                ModuleNode.generate_cfunction_declaration(
8800 8801 8802 8803
                    entry,
                    env.global_scope(),
                    h_code,
                    definition=True)
8804 8805 8806 8807 8808 8809 8810

                entry.cname = cname

        self.node.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
        self.node.generate_execution_code(code)
Mark Florisson's avatar
Mark Florisson committed
8811

8812

William Stein's avatar
William Stein committed
8813 8814 8815 8816 8817 8818
#------------------------------------------------------------------------------------
#
#  Runtime support code
#
#------------------------------------------------------------------------------------

Robert Bradshaw's avatar
Robert Bradshaw committed
8819
if Options.gcc_branch_hints:
Stefan Behnel's avatar
Stefan Behnel committed
8820
    branch_prediction_macros = """
8821 8822 8823 8824 8825 8826
/* Test for GCC > 2.95 */
#if defined(__GNUC__) \
    && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
  #define likely(x)   __builtin_expect(!!(x), 1)
  #define unlikely(x) __builtin_expect(!!(x), 0)
#else /* !__GNUC__ or GCC < 2.95 */
8827 8828
  #define likely(x)   (x)
  #define unlikely(x) (x)
8829
#endif /* __GNUC__ */
Stefan Behnel's avatar
Stefan Behnel committed
8830
"""
Robert Bradshaw's avatar
Robert Bradshaw committed
8831
else:
Stefan Behnel's avatar
Stefan Behnel committed
8832
    branch_prediction_macros = """
Robert Bradshaw's avatar
Robert Bradshaw committed
8833 8834
#define likely(x)   (x)
#define unlikely(x) (x)
Stefan Behnel's avatar
Stefan Behnel committed
8835
"""
William Stein's avatar
William Stein committed
8836 8837 8838

#------------------------------------------------------------------------------------

8839 8840
printing_utility_code = UtilityCode.load_cached("Print", "Printing.c")
printing_one_utility_code = UtilityCode.load_cached("PrintOne", "Printing.c")
8841

William Stein's avatar
William Stein committed
8842 8843
#------------------------------------------------------------------------------------

8844 8845 8846 8847 8848 8849 8850
# Exception raising code
#
# Exceptions are raised by __Pyx_Raise() and stored as plain
# type/value/tb in PyThreadState->curexc_*.  When being caught by an
# 'except' statement, curexc_* is moved over to exc_* by
# __Pyx_GetException()

8851 8852 8853 8854 8855
restore_exception_utility_code = UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")
raise_utility_code = UtilityCode.load_cached("RaiseException", "Exceptions.c")
get_exception_utility_code = UtilityCode.load_cached("GetException", "Exceptions.c")
swap_exception_utility_code = UtilityCode.load_cached("SwapException", "Exceptions.c")
reset_exception_utility_code = UtilityCode.load_cached("SaveResetException", "Exceptions.c")
8856
traceback_utility_code = UtilityCode.load_cached("AddTraceback", "Exceptions.c")
8857 8858 8859

#------------------------------------------------------------------------------------

8860 8861
get_exception_tuple_utility_code = UtilityCode(
    proto="""
8862
static PyObject *__Pyx_GetExceptionTuple(PyThreadState *__pyx_tstate); /*proto*/
8863
""",
8864 8865 8866 8867 8868
    # I doubt that calling __Pyx_GetException() here is correct as it moves
    # the exception from tstate->curexc_* to tstate->exc_*, which prevents
    # exception handlers later on from receiving it.
    # NOTE: "__pyx_tstate" may be used by __Pyx_GetException() macro
    impl = """
8869
static PyObject *__Pyx_GetExceptionTuple(CYTHON_UNUSED PyThreadState *__pyx_tstate) {
8870 8871 8872 8873 8874 8875 8876 8877 8878 8879 8880 8881 8882 8883 8884 8885
    PyObject *type = NULL, *value = NULL, *tb = NULL;
    if (__Pyx_GetException(&type, &value, &tb) == 0) {
        PyObject* exc_info = PyTuple_New(3);
        if (exc_info) {
            Py_INCREF(type);
            Py_INCREF(value);
            Py_INCREF(tb);
            PyTuple_SET_ITEM(exc_info, 0, type);
            PyTuple_SET_ITEM(exc_info, 1, value);
            PyTuple_SET_ITEM(exc_info, 2, tb);
            return exc_info;
        }
    }
    return NULL;
}
""",
8886
    requires=[get_exception_utility_code])