Nodes.py 341 KB
Newer Older
William Stein's avatar
William Stein committed
1
#
2
#   Parse tree nodes
William Stein's avatar
William Stein committed
3
#
4

5 6
from __future__ import absolute_import

7
import cython
8
cython.declare(sys=object, os=object, copy=object,
9
               Builtin=object, error=object, warning=object, Naming=object, PyrexTypes=object,
10
               py_object_type=object, ModuleScope=object, LocalScope=object, ClosureScope=object,
11
               StructOrUnionScope=object, PyClassScope=object,
12
               CppClassScope=object, UtilityCode=object, EncodedString=object,
13
               absolute_path_length=cython.Py_ssize_t, error_type=object)
William Stein's avatar
William Stein committed
14

15
import sys, os, copy
Stefan Behnel's avatar
Stefan Behnel committed
16
from itertools import chain
Robert Bradshaw's avatar
Robert Bradshaw committed
17

18 19 20 21 22 23 24
from . import Builtin
from .Errors import error, warning, InternalError, CompileError
from . import Naming
from . import PyrexTypes
from . import TypeSlots
from .PyrexTypes import py_object_type, error_type
from .Symtab import (ModuleScope, LocalScope, ClosureScope,
25
    StructOrUnionScope, PyClassScope, CppClassScope, TemplateScope)
26 27
from .Code import UtilityCode
from .StringEncoding import EncodedString, escape_byte_string, split_string_literal
28
from . import Future
29 30
from . import Options
from . import DebugFlags
William Stein's avatar
William Stein committed
31

Gary Furnish's avatar
Gary Furnish committed
32
absolute_path_length = 0
33

34

35 36 37
def relative_position(pos):
    """
    We embed the relative filename in the generated C file, since we
Stefan Behnel's avatar
Stefan Behnel committed
38
    don't want to have to regenerate and compile all the source code
39 40
    whenever the Python install directory moves (which could happen,
    e.g,. when distributing binaries.)
41

42 43 44 45 46 47 48 49 50
    INPUT:
        a position tuple -- (absolute filename, line number column position)

    OUTPUT:
        relative filename
        line number

    AUTHOR: William Stein
    """
Gary Furnish's avatar
Gary Furnish committed
51 52
    global absolute_path_length
    if absolute_path_length==0:
53
        absolute_path_length = len(os.path.abspath(os.getcwd()))
54
    return (pos[0].get_filenametable_entry()[absolute_path_length+1:], pos[1])
55 56 57 58

def embed_position(pos, docstring):
    if not Options.embed_pos_in_docstring:
        return docstring
59
    pos_line = u'File: %s (starting at line %s)' % relative_position(pos)
60 61
    if docstring is None:
        # unicode string
62
        return EncodedString(pos_line)
63 64 65 66 67 68

    # make sure we can encode the filename in the docstring encoding
    # otherwise make the docstring a unicode string
    encoding = docstring.encoding
    if encoding is not None:
        try:
Stefan Behnel's avatar
Stefan Behnel committed
69
            pos_line.encode(encoding)
70 71 72 73 74
        except UnicodeEncodeError:
            encoding = None

    if not docstring:
        # reuse the string encoding of the original docstring
75
        doc = EncodedString(pos_line)
76
    else:
77
        doc = EncodedString(pos_line + u'\n' + docstring)
78 79
    doc.encoding = encoding
    return doc
80

81

82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
def _analyse_signature_annotation(annotation, env):
    base_type = None
    explicit_pytype = explicit_ctype = False
    if annotation.is_dict_literal:
        for name, value in annotation.key_value_pairs:
            if not name.is_string_literal:
                continue
            if name.value in ('type', b'type'):
                explicit_pytype = True
                if not explicit_ctype:
                    annotation = value
            elif name.value in ('ctype', b'ctype'):
                explicit_ctype = True
                annotation = value
        if explicit_pytype and explicit_ctype:
            warning(annotation.pos, "Duplicate type declarations found in signature annotation")
    arg_type = annotation.analyse_as_type(env)
    if arg_type is not None:
        if explicit_pytype and not explicit_ctype and not arg_type.is_pyobject:
            warning(annotation.pos,
                    "Python type declaration in signature annotation does not refer to a Python type")
        base_type = CAnalysedBaseTypeNode(
            annotation.pos, type=arg_type, is_arg=True)
    else:
        warning(annotation.pos, "Unknown type declaration found in signature annotation")
    return base_type, arg_type


110
def write_func_call(func, codewriter_class):
111
    def f(*args, **kwds):
112
        if len(args) > 1 and isinstance(args[1], codewriter_class):
Robert Bradshaw's avatar
Robert Bradshaw committed
113 114
            # here we annotate the code with this function call
            # but only if new code is generated
115
            node, code = args[:2]
Robert Bradshaw's avatar
Robert Bradshaw committed
116
            marker = '                    /* %s -> %s.%s %s */' % (
Stefan Behnel's avatar
Stefan Behnel committed
117 118 119 120
                ' ' * code.call_level,
                node.__class__.__name__,
                func.__name__,
                node.pos[1:])
Robert Bradshaw's avatar
Robert Bradshaw committed
121 122 123 124 125 126 127
            pristine = code.buffer.stream.tell()
            code.putln(marker)
            start = code.buffer.stream.tell()
            code.call_level += 4
            res = func(*args, **kwds)
            code.call_level -= 4
            if start == code.buffer.stream.tell():
128 129
                # no code written => undo writing marker
                code.buffer.stream.truncate(pristine)
Robert Bradshaw's avatar
Robert Bradshaw committed
130
            else:
131
                marker = marker.replace('->', '<-', 1)
Robert Bradshaw's avatar
Robert Bradshaw committed
132 133 134 135
                code.putln(marker)
            return res
        else:
            return func(*args, **kwds)
136 137
    return f

138

139 140
class VerboseCodeWriter(type):
    # Set this as a metaclass to trace function calls in code.
141
    # This slows down code generation and makes much larger files.
142
    def __new__(cls, name, bases, attrs):
143
        from types import FunctionType
144
        from .Code import CCodeWriter
145 146 147
        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType):
148
                attrs[mname] = write_func_call(m, CCodeWriter)
149 150 151
        return super(VerboseCodeWriter, cls).__new__(cls, name, bases, attrs)


152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175
class CheckAnalysers(type):
    """Metaclass to check that type analysis functions return a node.
    """
    methods = set(['analyse_types',
                   'analyse_expressions',
                   'analyse_target_types'])

    def __new__(cls, name, bases, attrs):
        from types import FunctionType
        def check(name, func):
            def call(*args, **kwargs):
                retval = func(*args, **kwargs)
                if retval is None:
                    print name, args, kwargs
                return retval
            return call

        attrs = dict(attrs)
        for mname, m in attrs.items():
            if isinstance(m, FunctionType) and mname in cls.methods:
                attrs[mname] = check(mname, m)
        return super(CheckAnalysers, cls).__new__(cls, name, bases, attrs)


Stefan Behnel's avatar
Stefan Behnel committed
176
class Node(object):
William Stein's avatar
William Stein committed
177 178 179
    #  pos         (string, int, int)   Source file position
    #  is_name     boolean              Is a NameNode
    #  is_literal  boolean              Is a ConstNode
180

181
    #__metaclass__ = CheckAnalysers
182 183
    if DebugFlags.debug_trace_code_generation:
        __metaclass__ = VerboseCodeWriter
184

William Stein's avatar
William Stein committed
185
    is_name = 0
186
    is_none = 0
187
    is_nonecheck = 0
William Stein's avatar
William Stein committed
188
    is_literal = 0
189
    is_terminator = 0
190
    temps = None
191

Stefan Behnel's avatar
Stefan Behnel committed
192
    # All descendants should set child_attrs to a list of the attributes
193 194
    # containing nodes considered "children" in the tree. Each such attribute
    # can either contain a single node or a list of nodes. See Visitor.py.
195
    child_attrs = None
196

197 198
    cf_state = None

199 200 201 202 203
    # This may be an additional (or 'actual') type that will be checked when
    # this node is coerced to another type. This could be useful to set when
    # the actual type to which it can coerce is known, but you want to leave
    # the type a py_object_type
    coercion_type = None
204

William Stein's avatar
William Stein committed
205 206 207
    def __init__(self, pos, **kw):
        self.pos = pos
        self.__dict__.update(kw)
208

209 210
    gil_message = "Operation"

211
    nogil_check = None
212

213
    def gil_error(self, env=None):
214
        error(self.pos, "%s not allowed without gil" % self.gil_message)
215

Robert Bradshaw's avatar
Robert Bradshaw committed
216
    cpp_message = "Operation"
217

Robert Bradshaw's avatar
Robert Bradshaw committed
218 219 220 221 222 223
    def cpp_check(self, env):
        if not env.is_cpp():
            self.cpp_error()

    def cpp_error(self):
        error(self.pos, "%s only allowed in c++" % self.cpp_message)
224

225 226 227 228
    def clone_node(self):
        """Clone the node. This is defined as a shallow copy, except for member lists
           amongst the child attributes (from get_child_accessors) which are also
           copied. Lists containing child nodes are thus seen as a way for the node
Stefan Behnel's avatar
Stefan Behnel committed
229
           to hold multiple children directly; the list is not treated as a separate
230
           level in the tree."""
231 232 233
        result = copy.copy(self)
        for attrname in result.child_attrs:
            value = getattr(result, attrname)
234
            if isinstance(value, list):
235
                setattr(result, attrname, [x for x in value])
236
        return result
237 238


William Stein's avatar
William Stein committed
239
    #
240
    #  There are 3 phases of parse tree processing, applied in order to
William Stein's avatar
William Stein committed
241 242
    #  all the statements in a given scope-block:
    #
243
    #  (0) analyse_declarations
William Stein's avatar
William Stein committed
244 245 246 247
    #        Make symbol table entries for all declarations at the current
    #        level, both explicit (def, cdef, etc.) and implicit (assignment
    #        to an otherwise undeclared name).
    #
248
    #  (1) analyse_expressions
William Stein's avatar
William Stein committed
249 250
    #         Determine the result types of expressions and fill in the
    #         'type' attribute of each ExprNode. Insert coercion nodes into the
251
    #         tree where needed to convert to and from Python objects.
William Stein's avatar
William Stein committed
252 253 254 255
    #         Allocate temporary locals for intermediate results. Fill
    #         in the 'result_code' attribute of each ExprNode with a C code
    #         fragment.
    #
256
    #  (2) generate_code
William Stein's avatar
William Stein committed
257 258 259 260
    #         Emit C code for all declarations, statements and expressions.
    #         Recursively applies the 3 processing phases to the bodies of
    #         functions.
    #
261

William Stein's avatar
William Stein committed
262 263
    def analyse_declarations(self, env):
        pass
264

William Stein's avatar
William Stein committed
265 266 267
    def analyse_expressions(self, env):
        raise InternalError("analyse_expressions not implemented for %s" % \
            self.__class__.__name__)
268

William Stein's avatar
William Stein committed
269 270 271
    def generate_code(self, code):
        raise InternalError("generate_code not implemented for %s" % \
            self.__class__.__name__)
272

273 274 275 276
    def annotate(self, code):
        # mro does the wrong thing
        if isinstance(self, BlockNode):
            self.body.annotate(code)
277

278 279 280 281
    def end_pos(self):
        try:
            return self._end_pos
        except AttributeError:
Stefan Behnel's avatar
Stefan Behnel committed
282
            pos = self.pos
283 284 285
            if not self.child_attrs:
                self._end_pos = pos
                return pos
286
            for attr in self.child_attrs:
287
                child = getattr(self, attr)
288
                # Sometimes lists, sometimes nodes
289 290 291
                if child is None:
                    pass
                elif isinstance(child, list):
Stefan Behnel's avatar
Stefan Behnel committed
292 293
                    for c in child:
                        pos = max(pos, c.end_pos())
294
                else:
Stefan Behnel's avatar
Stefan Behnel committed
295 296 297
                    pos = max(pos, child.end_pos())
            self._end_pos = pos
            return pos
William Stein's avatar
William Stein committed
298

299
    def dump(self, level=0, filter_out=("pos",), cutoff=100, encountered=None):
300 301
        """Debug helper method that returns a recursive string representation of this node.
        """
302 303
        if cutoff == 0:
            return "<...nesting level cutoff...>"
304 305 306
        if encountered is None:
            encountered = set()
        if id(self) in encountered:
307
            return "<%s (0x%x) -- already output>" % (self.__class__.__name__, id(self))
308
        encountered.add(id(self))
309

310 311
        def dump_child(x, level):
            if isinstance(x, Node):
312
                return x.dump(level, filter_out, cutoff-1, encountered)
313
            elif isinstance(x, list):
Robert Bradshaw's avatar
Robert Bradshaw committed
314
                return "[%s]" % ", ".join([dump_child(item, level) for item in x])
315 316
            else:
                return repr(x)
317 318


319
        attrs = [(key, value) for key, value in self.__dict__.items() if key not in filter_out]
320
        if len(attrs) == 0:
321
            return "<%s (0x%x)>" % (self.__class__.__name__, id(self))
322 323
        else:
            indent = "  " * level
324
            res = "<%s (0x%x)\n" % (self.__class__.__name__, id(self))
325 326 327 328
            for key, value in attrs:
                res += "%s  %s: %s\n" % (indent, key, dump_child(value, level + 1))
            res += "%s>" % indent
            return res
329

330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347
    def dump_pos(self, mark_column=False, marker='(#)'):
        """Debug helper method that returns the source code context of this node as a string.
        """
        if not self.pos:
            return u''
        source_desc, line, col = self.pos
        contents = source_desc.get_lines(encoding='ASCII',
                                         error_handling='ignore')
        # line numbers start at 1
        lines = contents[max(0,line-3):line]
        current = lines[-1]
        if mark_column:
            current = current[:col] + marker + current[col:]
        lines[-1] = current.rstrip() + u'             # <<<<<<<<<<<<<<\n'
        lines += contents[line:line+2]
        return u'"%s":%d:%d\n%s\n' % (
            source_desc.get_escaped_description(), line, col, u''.join(lines))

348 349 350 351 352 353 354 355 356 357 358 359 360 361
class CompilerDirectivesNode(Node):
    """
    Sets compiler directives for the children nodes
    """
    #  directives     {string:value}  A dictionary holding the right value for
    #                                 *all* possible directives.
    #  body           Node
    child_attrs = ["body"]

    def analyse_declarations(self, env):
        old = env.directives
        env.directives = self.directives
        self.body.analyse_declarations(env)
        env.directives = old
362

363 364 365
    def analyse_expressions(self, env):
        old = env.directives
        env.directives = self.directives
366
        self.body = self.body.analyse_expressions(env)
367
        env.directives = old
368
        return self
369 370 371 372 373 374 375 376

    def generate_function_definitions(self, env, code):
        env_old = env.directives
        code_old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_function_definitions(env, code)
        env.directives = env_old
        code.globalstate.directives = code_old
377

378 379 380 381 382
    def generate_execution_code(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.generate_execution_code(code)
        code.globalstate.directives = old
383

384 385 386 387 388
    def annotate(self, code):
        old = code.globalstate.directives
        code.globalstate.directives = self.directives
        self.body.annotate(code)
        code.globalstate.directives = old
389

Stefan Behnel's avatar
Stefan Behnel committed
390
class BlockNode(object):
William Stein's avatar
William Stein committed
391 392
    #  Mixin class for nodes representing a declaration block.

393
    def generate_cached_builtins_decls(self, env, code):
394
        entries = env.global_scope().undeclared_cached_builtins
395
        for entry in entries:
396
            code.globalstate.add_cached_builtin_decl(entry)
397
        del entries[:]
398 399 400 401

    def generate_lambda_definitions(self, env, code):
        for node in env.lambda_defs:
            node.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
402 403 404

class StatListNode(Node):
    # stats     a list of StatNode
405

406
    child_attrs = ["stats"]
407 408 409 410 411

    def create_analysed(pos, env, *args, **kw):
        node = StatListNode(pos, *args, **kw)
        return node # No node-specific analysis necesarry
    create_analysed = staticmethod(create_analysed)
412

William Stein's avatar
William Stein committed
413 414
    def analyse_declarations(self, env):
        #print "StatListNode.analyse_declarations" ###
415
        for stat in self.stats:
William Stein's avatar
William Stein committed
416
            stat.analyse_declarations(env)
417

William Stein's avatar
William Stein committed
418 419
    def analyse_expressions(self, env):
        #print "StatListNode.analyse_expressions" ###
420 421 422
        self.stats = [ stat.analyse_expressions(env)
                       for stat in self.stats ]
        return self
423

424
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
425 426
        #print "StatListNode.generate_function_definitions" ###
        for stat in self.stats:
427
            stat.generate_function_definitions(env, code)
428

William Stein's avatar
William Stein committed
429 430 431 432 433
    def generate_execution_code(self, code):
        #print "StatListNode.generate_execution_code" ###
        for stat in self.stats:
            code.mark_pos(stat.pos)
            stat.generate_execution_code(code)
434

435 436 437
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)
438

William Stein's avatar
William Stein committed
439 440 441 442 443 444 445 446 447 448 449 450 451

class StatNode(Node):
    #
    #  Code generation for statements is split into the following subphases:
    #
    #  (1) generate_function_definitions
    #        Emit C code for the definitions of any structs,
    #        unions, enums and functions defined in the current
    #        scope-block.
    #
    #  (2) generate_execution_code
    #        Emit C code for executable statements.
    #
452

453
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
454
        pass
455

William Stein's avatar
William Stein committed
456 457 458 459 460 461 462 463
    def generate_execution_code(self, code):
        raise InternalError("generate_execution_code not implemented for %s" % \
            self.__class__.__name__)


class CDefExternNode(StatNode):
    #  include_file   string or None
    #  body           StatNode
464

465
    child_attrs = ["body"]
466

William Stein's avatar
William Stein committed
467 468 469 470 471 472 473
    def analyse_declarations(self, env):
        if self.include_file:
            env.add_include_file(self.include_file)
        old_cinclude_flag = env.in_cinclude
        env.in_cinclude = 1
        self.body.analyse_declarations(env)
        env.in_cinclude = old_cinclude_flag
474

William Stein's avatar
William Stein committed
475
    def analyse_expressions(self, env):
476
        return self
477

William Stein's avatar
William Stein committed
478 479
    def generate_execution_code(self, code):
        pass
480 481 482

    def annotate(self, code):
        self.body.annotate(code)
483

William Stein's avatar
William Stein committed
484 485 486 487 488 489 490 491

class CDeclaratorNode(Node):
    # Part of a C declaration.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #      Returns (name, type) pair where name is the
492
    #      CNameDeclaratorNode of the name being declared
William Stein's avatar
William Stein committed
493 494
    #      and type is the type it is being declared as.
    #
495
    #  calling_convention  string   Calling convention of CFuncDeclaratorNode
496
    #                               for which this is a base
497

498 499
    child_attrs = []

500 501
    calling_convention = ""

502 503 504
    def analyse_templates(self):
        # Only C++ functions have templates.
        return None
William Stein's avatar
William Stein committed
505 506

class CNameDeclaratorNode(CDeclaratorNode):
507
    #  name    string             The Cython name being declared
Robert Bradshaw's avatar
Robert Bradshaw committed
508 509
    #  cname   string or None     C name, if specified
    #  default ExprNode or None   the value assigned on declaration
510

Robert Bradshaw's avatar
Robert Bradshaw committed
511
    child_attrs = ['default']
512

513
    default = None
514

515 516
    def analyse(self, base_type, env, nonempty = 0):
        if nonempty and self.name == '':
517
            # May have mistaken the name for the type.
518
            if base_type.is_ptr or base_type.is_array or base_type.is_buffer:
519
                error(self.pos, "Missing argument name")
520 521
            elif base_type.is_void:
                error(self.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
522 523 524
            else:
                self.name = base_type.declaration_code("", for_display=1, pyrex=1)
                base_type = py_object_type
525 526 527 528

        if base_type.is_fused and env.fused_to_specific:
            base_type = base_type.specialize(env.fused_to_specific)

529
        self.type = base_type
William Stein's avatar
William Stein committed
530
        return self, base_type
531

William Stein's avatar
William Stein committed
532 533
class CPtrDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode
534

535 536
    child_attrs = ["base"]

537
    def analyse(self, base_type, env, nonempty = 0):
William Stein's avatar
William Stein committed
538 539 540 541
        if base_type.is_pyobject:
            error(self.pos,
                "Pointer base type cannot be a Python object")
        ptr_type = PyrexTypes.c_ptr_type(base_type)
542
        return self.base.analyse(ptr_type, env, nonempty = nonempty)
Danilo Freitas's avatar
Danilo Freitas committed
543 544 545 546 547 548 549 550 551 552

class CReferenceDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

    def analyse(self, base_type, env, nonempty = 0):
        if base_type.is_pyobject:
            error(self.pos,
                  "Reference base type cannot be a Python object")
553
        ref_type = PyrexTypes.c_ref_type(base_type)
Danilo Freitas's avatar
Danilo Freitas committed
554 555
        return self.base.analyse(ref_type, env, nonempty = nonempty)

William Stein's avatar
William Stein committed
556 557 558
class CArrayDeclaratorNode(CDeclaratorNode):
    # base        CDeclaratorNode
    # dimension   ExprNode
559 560

    child_attrs = ["base", "dimension"]
561

562
    def analyse(self, base_type, env, nonempty = 0):
563
        if (base_type.is_cpp_class and base_type.is_template_type()) or base_type.is_cfunction:
564
            from .ExprNodes import TupleNode
Robert Bradshaw's avatar
Robert Bradshaw committed
565 566 567 568 569 570 571
            if isinstance(self.dimension, TupleNode):
                args = self.dimension.args
            else:
                args = self.dimension,
            values = [v.analyse_as_type(env) for v in args]
            if None in values:
                ix = values.index(None)
572 573 574 575
                error(args[ix].pos, "Template parameter not a type")
                base_type = error_type
            else:
                base_type = base_type.specialize_here(self.pos, values)
Robert Bradshaw's avatar
Robert Bradshaw committed
576
            return self.base.analyse(base_type, env, nonempty = nonempty)
William Stein's avatar
William Stein committed
577
        if self.dimension:
578
            self.dimension = self.dimension.analyse_const_expression(env)
William Stein's avatar
William Stein committed
579 580
            if not self.dimension.type.is_int:
                error(self.dimension.pos, "Array dimension not integer")
581 582 583 584 585 586 587
            size = self.dimension.get_constant_c_result_code()
            if size is not None:
                try:
                    size = int(size)
                except ValueError:
                    # runtime constant?
                    pass
William Stein's avatar
William Stein committed
588 589 590 591 592 593 594 595
        else:
            size = None
        if not base_type.is_complete():
            error(self.pos,
                "Array element type '%s' is incomplete" % base_type)
        if base_type.is_pyobject:
            error(self.pos,
                "Array element cannot be a Python object")
596 597 598
        if base_type.is_cfunction:
            error(self.pos,
                "Array element cannot be a function")
William Stein's avatar
William Stein committed
599
        array_type = PyrexTypes.c_array_type(base_type, size)
600
        return self.base.analyse(array_type, env, nonempty = nonempty)
William Stein's avatar
William Stein committed
601 602 603 604 605


class CFuncDeclaratorNode(CDeclaratorNode):
    # base             CDeclaratorNode
    # args             [CArgDeclNode]
606
    # templates        [TemplatePlaceholderType]
William Stein's avatar
William Stein committed
607 608 609
    # has_varargs      boolean
    # exception_value  ConstNode
    # exception_check  boolean    True if PyErr_Occurred check needed
610 611
    # nogil            boolean    Can be called without gil
    # with_gil         boolean    Acquire gil around function body
612
    # is_const_method  boolean    Whether this is a const method
613

614 615
    child_attrs = ["base", "args", "exception_value"]

616
    overridable = 0
617
    optional_arg_count = 0
618
    is_const_method = 0
619 620 621 622
    templates = None

    def analyse_templates(self):
        if isinstance(self.base, CArrayDeclaratorNode):
623
            from .ExprNodes import TupleNode, NameNode
624 625 626 627 628 629 630
            template_node = self.base.dimension
            if isinstance(template_node, TupleNode):
                template_nodes = template_node.args
            elif isinstance(template_node, NameNode):
                template_nodes = [template_node]
            else:
                error(template_node.pos, "Template arguments must be a list of names")
631
                return None
632 633 634 635 636 637 638 639 640 641
            self.templates = []
            for template in template_nodes:
                if isinstance(template, NameNode):
                    self.templates.append(PyrexTypes.TemplatePlaceholderType(template.name))
                else:
                    error(template.pos, "Template arguments must be a list of names")
            self.base = self.base.base
            return self.templates
        else:
            return None
William Stein's avatar
William Stein committed
642

643
    def analyse(self, return_type, env, nonempty = 0, directive_locals = {}):
644 645
        if nonempty:
            nonempty -= 1
William Stein's avatar
William Stein committed
646
        func_type_args = []
647
        for i, arg_node in enumerate(self.args):
648
            name_declarator, type = arg_node.analyse(
Robert Bradshaw's avatar
Robert Bradshaw committed
649
                env, nonempty=nonempty, is_self_arg=(i == 0 and env.is_c_class_scope and 'staticmethod' not in env.directives))
William Stein's avatar
William Stein committed
650
            name = name_declarator.name
651 652 653 654 655
            if name in directive_locals:
                type_node = directive_locals[name]
                other_type = type_node.analyse_as_type(env)
                if other_type is None:
                    error(type_node.pos, "Not a type")
Robert Bradshaw's avatar
Robert Bradshaw committed
656
                elif (type is not PyrexTypes.py_object_type
657 658 659 660 661
                      and not type.same_as(other_type)):
                    error(self.base.pos, "Signature does not agree with previous declaration")
                    error(type_node.pos, "Previous declaration here")
                else:
                    type = other_type
William Stein's avatar
William Stein committed
662
            if name_declarator.cname:
663
                error(self.pos,
William Stein's avatar
William Stein committed
664
                    "Function argument cannot have C name specification")
665 666 667
            if i==0 and env.is_c_class_scope and type.is_unspecified:
                # fix the type of self
                type = env.parent_type
William Stein's avatar
William Stein committed
668 669 670 671 672
            # Turn *[] argument into **
            if type.is_array:
                type = PyrexTypes.c_ptr_type(type.base_type)
            # Catch attempted C-style func(void) decl
            if type.is_void:
Robert Bradshaw's avatar
Robert Bradshaw committed
673
                error(arg_node.pos, "Use spam() rather than spam(void) to declare a function with no arguments.")
William Stein's avatar
William Stein committed
674 675 676
            func_type_args.append(
                PyrexTypes.CFuncTypeArg(name, type, arg_node.pos))
            if arg_node.default:
677
                self.optional_arg_count += 1
678 679
            elif self.optional_arg_count:
                error(self.pos, "Non-default argument follows default argument")
680

William Stein's avatar
William Stein committed
681 682
        exc_val = None
        exc_check = 0
683
        if self.exception_check == '+':
684
            env.add_include_file('ios')         # for std::ios_base::failure
685
            env.add_include_file('new')         # for std::bad_alloc
686
            env.add_include_file('stdexcept')
687
            env.add_include_file('typeinfo')    # for std::bad_cast
688 689 690 691 692
        if (return_type.is_pyobject
                and (self.exception_value or self.exception_check)
                and self.exception_check != '+'):
            error(self.pos,
                "Exception clause not allowed for function returning Python object")
William Stein's avatar
William Stein committed
693 694
        else:
            if self.exception_value:
695
                self.exception_value = self.exception_value.analyse_const_expression(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
696 697
                if self.exception_check == '+':
                    exc_val_type = self.exception_value.type
698 699 700 701 702
                    if (not exc_val_type.is_error
                            and not exc_val_type.is_pyobject
                            and not (exc_val_type.is_cfunction
                                     and not exc_val_type.return_type.is_pyobject
                                     and not exc_val_type.args)):
Robert Bradshaw's avatar
Robert Bradshaw committed
703
                        error(self.exception_value.pos,
704
                              "Exception value must be a Python exception or cdef function with no arguments.")
705
                    exc_val = self.exception_value
Robert Bradshaw's avatar
Robert Bradshaw committed
706
                else:
707 708 709 710 711 712 713 714 715 716
                    self.exception_value = self.exception_value.coerce_to(
                        return_type, env).analyse_const_expression(env)
                    exc_val = self.exception_value.get_constant_c_result_code()
                    if exc_val is None:
                        raise InternalError(
                            "get_constant_c_result_code not implemented for %s" %
                            self.exception_value.__class__.__name__)
                    if not return_type.assignable_from(self.exception_value.type):
                        error(self.exception_value.pos,
                              "Exception value incompatible with function return type")
William Stein's avatar
William Stein committed
717
            exc_check = self.exception_check
718 719 720
        if return_type.is_cfunction:
            error(self.pos,
                "Function cannot return a function")
William Stein's avatar
William Stein committed
721
        func_type = PyrexTypes.CFuncType(
722
            return_type, func_type_args, self.has_varargs,
723
            optional_arg_count = self.optional_arg_count,
724
            exception_value = exc_val, exception_check = exc_check,
725
            calling_convention = self.base.calling_convention,
726
            nogil = self.nogil, with_gil = self.with_gil, is_overridable = self.overridable,
727
            is_const_method = self.is_const_method,
728
            templates = self.templates)
729

730
        if self.optional_arg_count:
731 732 733 734 735 736 737 738 739 740 741
            if func_type.is_fused:
                # This is a bit of a hack... When we need to create specialized CFuncTypes
                # on the fly because the cdef is defined in a pxd, we need to declare the specialized optional arg
                # struct
                def declare_opt_arg_struct(func_type, fused_cname):
                    self.declare_optional_arg_struct(func_type, env, fused_cname)

                func_type.declare_opt_arg_struct = declare_opt_arg_struct
            else:
                self.declare_optional_arg_struct(func_type, env)

742 743 744 745 746 747 748
        callspec = env.directives['callspec']
        if callspec:
            current = func_type.calling_convention
            if current and current != callspec:
                error(self.pos, "cannot have both '%s' and '%s' "
                      "calling conventions" % (current, callspec))
            func_type.calling_convention = callspec
William Stein's avatar
William Stein committed
749 750
        return self.base.analyse(func_type, env)

751 752 753 754 755 756 757 758 759 760 761 762 763 764 765 766 767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782
    def declare_optional_arg_struct(self, func_type, env, fused_cname=None):
        """
        Declares the optional argument struct (the struct used to hold the
        values for optional arguments). For fused cdef functions, this is
        deferred as analyse_declarations is called only once (on the fused
        cdef function).
        """
        scope = StructOrUnionScope()
        arg_count_member = '%sn' % Naming.pyrex_prefix
        scope.declare_var(arg_count_member, PyrexTypes.c_int_type, self.pos)

        for arg in func_type.args[len(func_type.args)-self.optional_arg_count:]:
            scope.declare_var(arg.name, arg.type, arg.pos, allow_pyobject = 1)

        struct_cname = env.mangle(Naming.opt_arg_prefix, self.base.name)

        if fused_cname is not None:
            struct_cname = PyrexTypes.get_fused_cname(fused_cname, struct_cname)

        op_args_struct = env.global_scope().declare_struct_or_union(
                name = struct_cname,
                kind = 'struct',
                scope = scope,
                typedef_flag = 0,
                pos = self.pos,
                cname = struct_cname)

        op_args_struct.defined_in_pxd = 1
        op_args_struct.used = 1

        func_type.op_arg_struct = PyrexTypes.c_ptr_type(op_args_struct.type)

William Stein's avatar
William Stein committed
783

784 785 786 787 788 789 790 791 792 793 794 795 796
class CConstDeclaratorNode(CDeclaratorNode):
    # base     CDeclaratorNode

    child_attrs = ["base"]

    def analyse(self, base_type, env, nonempty = 0):
        if base_type.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        const = PyrexTypes.c_const_type(base_type)
        return self.base.analyse(const, env, nonempty = nonempty)


William Stein's avatar
William Stein committed
797 798 799 800 801 802
class CArgDeclNode(Node):
    # Item in a function declaration argument list.
    #
    # base_type      CBaseTypeNode
    # declarator     CDeclaratorNode
    # not_none       boolean            Tagged with 'not None'
803 804
    # or_none        boolean            Tagged with 'or None'
    # accept_none    boolean            Resolved boolean for not_none/or_none
William Stein's avatar
William Stein committed
805
    # default        ExprNode or None
806
    # default_value  PyObjectConst      constant for default value
807
    # annotation     ExprNode or None   Py3 function arg annotation
William Stein's avatar
William Stein committed
808
    # is_self_arg    boolean            Is the "self" arg of an extension type method
809
    # is_type_arg    boolean            Is the "class" arg of an extension type classmethod
810
    # is_kw_only     boolean            Is a keyword-only argument
811
    # is_dynamic     boolean            Non-literal arg stored inside CyFunction
812

813
    child_attrs = ["base_type", "declarator", "default", "annotation"]
814

William Stein's avatar
William Stein committed
815
    is_self_arg = 0
816
    is_type_arg = 0
817
    is_generic = 1
818 819 820
    kw_only = 0
    not_none = 0
    or_none = 0
821 822
    type = None
    name_declarator = None
823
    default_value = None
824
    annotation = None
825
    is_dynamic = 0
826

827 828 829
    def analyse(self, env, nonempty = 0, is_self_arg = False):
        if is_self_arg:
            self.base_type.is_self_arg = self.is_self_arg = True
830
        if self.type is None:
Stefan Behnel's avatar
Stefan Behnel committed
831
            # The parser may misinterpret names as types. We fix that here.
832 833
            if isinstance(self.declarator, CNameDeclaratorNode) and self.declarator.name == '':
                if nonempty:
834 835
                    if self.base_type.is_basic_c_type:
                        # char, short, long called "int"
836
                        type = self.base_type.analyse(env, could_be_name=True)
837
                        arg_name = type.empty_declaration_code()
838 839 840
                    else:
                        arg_name = self.base_type.name
                    self.declarator.name = EncodedString(arg_name)
841 842 843 844 845
                    self.base_type.name = None
                    self.base_type.is_basic_c_type = False
                could_be_name = True
            else:
                could_be_name = False
846
            self.base_type.is_arg = True
847
            base_type = self.base_type.analyse(env, could_be_name=could_be_name)
Robert Bradshaw's avatar
Robert Bradshaw committed
848
            if hasattr(self.base_type, 'arg_name') and self.base_type.arg_name:
849
                self.declarator.name = self.base_type.arg_name
850

851 852
            # The parser is unable to resolve the ambiguity of [] as part of the
            # type (e.g. in buffers) or empty declarator (as with arrays).
853
            # This is only arises for empty multi-dimensional arrays.
854 855
            if (base_type.is_array
                    and isinstance(self.base_type, TemplatedTypeNode)
856 857 858 859 860 861
                    and isinstance(self.declarator, CArrayDeclaratorNode)):
                declarator = self.declarator
                while isinstance(declarator.base, CArrayDeclaratorNode):
                    declarator = declarator.base
                declarator.base = self.base_type.array_declarator
                base_type = base_type.base_type
862 863 864 865 866 867 868

            # inject type declaration from annotations
            if self.annotation and env.directives['annotation_typing'] and self.base_type.name is None:
                arg_type = self.inject_type_from_annotations(env)
                if arg_type is not None:
                    base_type = arg_type
            return self.declarator.analyse(base_type, env, nonempty=nonempty)
869
        else:
870
            return self.name_declarator, self.type
William Stein's avatar
William Stein committed
871

872 873 874
    def inject_type_from_annotations(self, env):
        annotation = self.annotation
        if not annotation:
875
            return None
876 877 878
        base_type, arg_type = _analyse_signature_annotation(annotation, env)
        if base_type is not None:
            self.base_type = base_type
879 880
        return arg_type

881 882 883 884 885 886 887 888 889
    def calculate_default_value_code(self, code):
        if self.default_value is None:
            if self.default:
                if self.default.is_literal:
                    # will not output any code, just assign the result_code
                    self.default.generate_evaluation_code(code)
                    return self.type.cast_code(self.default.result())
                self.default_value = code.get_argument_default_const(self.type)
        return self.default_value
890

891 892 893 894
    def annotate(self, code):
        if self.default:
            self.default.annotate(code)

895 896 897 898 899 900 901 902 903 904 905 906 907 908 909
    def generate_assignment_code(self, code, target=None):
        default = self.default
        if default is None or default.is_literal:
            return
        if target is None:
            target = self.calculate_default_value_code(code)
        default.generate_evaluation_code(code)
        default.make_owned_reference(code)
        result = default.result_as(self.type)
        code.putln("%s = %s;" % (target, result))
        if self.type.is_pyobject:
            code.put_giveref(default.result())
        default.generate_post_assignment_code(code)
        default.free_temps(code)

William Stein's avatar
William Stein committed
910 911 912 913 914 915 916 917

class CBaseTypeNode(Node):
    # Abstract base class for C base type nodes.
    #
    # Processing during analyse_declarations phase:
    #
    #   analyse
    #     Returns the type.
918

919 920
    def analyse_as_type(self, env):
        return self.analyse(env)
921

Stefan Behnel's avatar
Stefan Behnel committed
922

923 924
class CAnalysedBaseTypeNode(Node):
    # type            type
925

926
    child_attrs = []
927

928 929
    def analyse(self, env, could_be_name = False):
        return self.type
William Stein's avatar
William Stein committed
930

Stefan Behnel's avatar
Stefan Behnel committed
931

William Stein's avatar
William Stein committed
932 933 934 935 936 937
class CSimpleBaseTypeNode(CBaseTypeNode):
    # name             string
    # module_path      [string]     Qualifying name components
    # is_basic_c_type  boolean
    # signed           boolean
    # longness         integer
938
    # complex          boolean
William Stein's avatar
William Stein committed
939
    # is_self_arg      boolean      Is self argument of C method
940
    # ##is_type_arg      boolean      Is type argument of class method
William Stein's avatar
William Stein committed
941

942
    child_attrs = []
943
    arg_name = None   # in case the argument name was interpreted as a type
944 945 946
    module_path = []
    is_basic_c_type = False
    complex = False
947

948
    def analyse(self, env, could_be_name = False):
William Stein's avatar
William Stein committed
949
        # Return type descriptor.
950
        #print "CSimpleBaseTypeNode.analyse: is_self_arg =", self.is_self_arg ###
William Stein's avatar
William Stein committed
951 952 953 954 955 956 957 958 959
        type = None
        if self.is_basic_c_type:
            type = PyrexTypes.simple_c_type(self.signed, self.longness, self.name)
            if not type:
                error(self.pos, "Unrecognised type modifier combination")
        elif self.name == "object" and not self.module_path:
            type = py_object_type
        elif self.name is None:
            if self.is_self_arg and env.is_c_class_scope:
960
                #print "CSimpleBaseTypeNode.analyse: defaulting to parent type" ###
William Stein's avatar
William Stein committed
961
                type = env.parent_type
962 963
            ## elif self.is_type_arg and env.is_c_class_scope:
            ##     type = Builtin.type_type
William Stein's avatar
William Stein committed
964 965 966
            else:
                type = py_object_type
        else:
967
            if self.module_path:
968 969 970 971
                # Maybe it's a nested C++ class.
                scope = env
                for item in self.module_path:
                    entry = scope.lookup(item)
Robert Bradshaw's avatar
Robert Bradshaw committed
972
                    if entry is not None and entry.is_cpp_class:
973 974 975 976
                        scope = entry.type.scope
                    else:
                        scope = None
                        break
977

978 979 980 981 982
                if scope is None:
                    # Maybe it's a cimport.
                    scope = env.find_imported_module(self.module_path, self.pos)
                    if scope:
                        scope.fused_to_specific = env.fused_to_specific
983 984
            else:
                scope = env
985

William Stein's avatar
William Stein committed
986
            if scope:
987 988
                if scope.is_c_class_scope:
                    scope = scope.global_scope()
989 990 991 992

                type = scope.lookup_type(self.name)
                if type is not None:
                    pass
993 994 995
                elif could_be_name:
                    if self.is_self_arg and env.is_c_class_scope:
                        type = env.parent_type
996 997
                    ## elif self.is_type_arg and env.is_c_class_scope:
                    ##     type = Builtin.type_type
998 999
                    else:
                        type = py_object_type
1000
                    self.arg_name = EncodedString(self.name)
William Stein's avatar
William Stein committed
1001
                else:
Danilo Freitas's avatar
Danilo Freitas committed
1002 1003 1004
                    if self.templates:
                        if not self.name in self.templates:
                            error(self.pos, "'%s' is not a type identifier" % self.name)
1005
                        type = PyrexTypes.TemplatePlaceholderType(self.name)
Danilo Freitas's avatar
Danilo Freitas committed
1006 1007
                    else:
                        error(self.pos, "'%s' is not a type identifier" % self.name)
1008
        if self.complex:
1009 1010 1011
            if not type.is_numeric or type.is_complex:
                error(self.pos, "can only complexify c numeric types")
            type = PyrexTypes.CComplexType(type)
1012
            type.create_declaration_utility_code(env)
1013 1014 1015 1016 1017 1018 1019 1020
        elif type is Builtin.complex_type:
            # Special case: optimise builtin complex type into C's
            # double complex.  The parser cannot do this (as for the
            # normal scalar types) as the user may have redeclared the
            # 'complex' type.  Testing for the exact type here works.
            type = PyrexTypes.c_double_complex_type
            type.create_declaration_utility_code(env)
            self.complex = True
William Stein's avatar
William Stein committed
1021 1022 1023 1024 1025
        if type:
            return type
        else:
            return PyrexTypes.error_type

1026
class MemoryViewSliceTypeNode(CBaseTypeNode):
1027

1028
    name = 'memoryview'
1029 1030 1031 1032 1033 1034 1035
    child_attrs = ['base_type_node', 'axes']

    def analyse(self, env, could_be_name = False):

        base_type = self.base_type_node.analyse(env)
        if base_type.is_error: return base_type

1036
        from . import MemoryView
1037 1038 1039 1040 1041 1042 1043 1044

        try:
            axes_specs = MemoryView.get_axes_specs(env, self.axes)
        except CompileError, e:
            error(e.position, e.message_only)
            self.type = PyrexTypes.ErrorType()
            return self.type

1045 1046 1047 1048 1049 1050
        if not MemoryView.validate_axes(self.pos, axes_specs):
            self.type = error_type
        else:
            MemoryView.validate_memslice_dtype(self.pos, base_type)
            self.type = PyrexTypes.MemoryViewSliceType(base_type, axes_specs)
            self.use_memview_utilities(env)
1051

1052
        return self.type
1053

1054
    def use_memview_utilities(self, env):
1055
        from . import MemoryView
1056 1057 1058
        env.use_utility_code(MemoryView.view_utility_code)


Robert Bradshaw's avatar
Robert Bradshaw committed
1059
class CNestedBaseTypeNode(CBaseTypeNode):
1060
    # For C++ classes that live inside other C++ classes.
Robert Bradshaw's avatar
Robert Bradshaw committed
1061 1062 1063

    # name             string
    # base_type        CBaseTypeNode
Stefan Behnel's avatar
Stefan Behnel committed
1064

Robert Bradshaw's avatar
Robert Bradshaw committed
1065
    child_attrs = ['base_type']
Stefan Behnel's avatar
Stefan Behnel committed
1066

Robert Bradshaw's avatar
Robert Bradshaw committed
1067 1068 1069 1070 1071 1072 1073 1074 1075 1076 1077 1078 1079
    def analyse(self, env, could_be_name = None):
        base_type = self.base_type.analyse(env)
        if base_type is PyrexTypes.error_type:
            return PyrexTypes.error_type
        if not base_type.is_cpp_class:
            error(self.pos, "'%s' is not a valid type scope" % base_type)
            return PyrexTypes.error_type
        type_entry = base_type.scope.lookup_here(self.name)
        if not type_entry or not type_entry.is_type:
            error(self.pos, "'%s.%s' is not a type identifier" % (base_type, self.name))
            return PyrexTypes.error_type
        return type_entry.type

1080

Danilo Freitas's avatar
Danilo Freitas committed
1081
class TemplatedTypeNode(CBaseTypeNode):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1082
    #  After parsing:
1083 1084
    #  positional_args  [ExprNode]        List of positional arguments
    #  keyword_args     DictNode          Keyword arguments
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1085 1086 1087
    #  base_type_node   CBaseTypeNode

    #  After analysis:
1088
    #  type             PyrexTypes.BufferType or PyrexTypes.CppClassType  ...containing the right options
1089 1090 1091

    child_attrs = ["base_type_node", "positional_args",
                   "keyword_args", "dtype_node"]
1092

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1093
    dtype_node = None
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1094 1095

    name = None
1096

1097 1098 1099
    def analyse(self, env, could_be_name = False, base_type = None):
        if base_type is None:
            base_type = self.base_type_node.analyse(env)
1100
        if base_type.is_error: return base_type
1101

1102
        if base_type.is_cpp_class and base_type.is_template_type():
1103
            # Templated class
1104
            if self.keyword_args and self.keyword_args.key_value_pairs:
Stefan Behnel's avatar
Stefan Behnel committed
1105
                error(self.pos, "c++ templates cannot take keyword arguments")
1106 1107 1108 1109
                self.type = PyrexTypes.error_type
            else:
                template_types = []
                for template_node in self.positional_args:
1110 1111 1112 1113 1114
                    type = template_node.analyse_as_type(env)
                    if type is None:
                        error(template_node.pos, "unknown type in template argument")
                        return error_type
                    template_types.append(type)
1115
                self.type = base_type.specialize_here(self.pos, template_types)
1116

1117 1118
        elif base_type.is_pyobject:
            # Buffer
1119
            from . import Buffer
1120

1121 1122 1123 1124 1125 1126
            options = Buffer.analyse_buffer_options(
                self.pos,
                env,
                self.positional_args,
                self.keyword_args,
                base_type.buffer_defaults)
1127

Robert Bradshaw's avatar
Robert Bradshaw committed
1128 1129 1130
            if sys.version_info[0] < 3:
                # Py 2.x enforces byte strings as keyword arguments ...
                options = dict([ (name.encode('ASCII'), value)
1131
                                 for name, value in options.items() ])
Stefan Behnel's avatar
Stefan Behnel committed
1132

1133
            self.type = PyrexTypes.BufferType(base_type, **options)
1134

1135 1136
        else:
            # Array
1137
            empty_declarator = CNameDeclaratorNode(self.pos, name="", cname=None)
1138 1139 1140 1141
            if len(self.positional_args) > 1 or self.keyword_args.key_value_pairs:
                error(self.pos, "invalid array declaration")
                self.type = PyrexTypes.error_type
            else:
1142
                # It would be nice to merge this class with CArrayDeclaratorNode,
1143 1144 1145 1146 1147
                # but arrays are part of the declaration, not the type...
                if not self.positional_args:
                    dimension = None
                else:
                    dimension = self.positional_args[0]
1148 1149
                self.array_declarator = CArrayDeclaratorNode(self.pos,
                    base = empty_declarator,
1150 1151
                    dimension = dimension)
                self.type = self.array_declarator.analyse(base_type, env)[1]
1152

1153 1154 1155
        if self.type.is_fused and env.fused_to_specific:
            self.type = self.type.specialize(env.fused_to_specific)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1156
        return self.type
William Stein's avatar
William Stein committed
1157 1158 1159 1160

class CComplexBaseTypeNode(CBaseTypeNode):
    # base_type   CBaseTypeNode
    # declarator  CDeclaratorNode
1161

1162 1163
    child_attrs = ["base_type", "declarator"]

1164 1165
    def analyse(self, env, could_be_name = False):
        base = self.base_type.analyse(env, could_be_name)
William Stein's avatar
William Stein committed
1166 1167 1168 1169
        _, type = self.declarator.analyse(base, env)
        return type


1170 1171 1172 1173 1174
class CTupleBaseTypeNode(CBaseTypeNode):
    # components [CBaseTypeNode]

    child_attrs = ["components"]

Robert Bradshaw's avatar
Robert Bradshaw committed
1175
    def analyse(self, env, could_be_name=False):
1176 1177 1178 1179
        component_types = []
        for c in self.components:
            type = c.analyse(env)
            if type.is_pyobject:
1180 1181
                error(c.pos, "Tuple types can't (yet) contain Python objects.")
                return error_type
1182
            component_types.append(type)
Robert Bradshaw's avatar
Robert Bradshaw committed
1183
        type = PyrexTypes.c_tuple_type(component_types)
Robert Bradshaw's avatar
Robert Bradshaw committed
1184 1185
        entry = env.declare_tuple_type(self.pos, type)
        entry.used = True
1186 1187 1188
        return type


1189 1190 1191 1192 1193 1194
class FusedTypeNode(CBaseTypeNode):
    """
    Represents a fused type in a ctypedef statement:

        ctypedef cython.fused_type(int, long, long long) integral

Mark Florisson's avatar
Mark Florisson committed
1195
    name            str                     name of this fused type
1196 1197 1198 1199 1200
    types           [CSimpleBaseTypeNode]   is the list of types to be fused
    """

    child_attrs = []

Mark Florisson's avatar
Mark Florisson committed
1201 1202 1203 1204 1205 1206 1207
    def analyse_declarations(self, env):
        type = self.analyse(env)
        entry = env.declare_typedef(self.name, type, self.pos)

        # Omit the typedef declaration that self.declarator would produce
        entry.in_cinclude = True

Robert Bradshaw's avatar
Robert Bradshaw committed
1208
    def analyse(self, env, could_be_name = False):
Mark Florisson's avatar
Mark Florisson committed
1209 1210 1211
        types = []
        for type_node in self.types:
            type = type_node.analyse_as_type(env)
1212

Mark Florisson's avatar
Mark Florisson committed
1213 1214 1215
            if not type:
                error(type_node.pos, "Not a type")
                continue
1216

Mark Florisson's avatar
Mark Florisson committed
1217
            if type in types:
1218
                error(type_node.pos, "Type specified multiple times")
Mark Florisson's avatar
Mark Florisson committed
1219 1220
            elif type.is_fused:
                error(type_node.pos, "Cannot fuse a fused type")
1221
            else:
Mark Florisson's avatar
Mark Florisson committed
1222
                types.append(type)
1223

1224 1225
        # if len(self.types) == 1:
        #     return types[0]
Mark Florisson's avatar
Mark Florisson committed
1226 1227

        return PyrexTypes.FusedType(types, name=self.name)
1228

1229

Robert Bradshaw's avatar
Robert Bradshaw committed
1230 1231 1232 1233 1234 1235 1236 1237 1238 1239 1240 1241 1242
class CConstTypeNode(CBaseTypeNode):
    # base_type     CBaseTypeNode

    child_attrs = ["base_type"]

    def analyse(self, env, could_be_name = False):
        base = self.base_type.analyse(env, could_be_name)
        if base.is_pyobject:
            error(self.pos,
                  "Const base type cannot be a Python object")
        return PyrexTypes.c_const_type(base)


William Stein's avatar
William Stein committed
1243 1244 1245 1246 1247 1248
class CVarDefNode(StatNode):
    #  C variable definition or forward/extern function declaration.
    #
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarators   [CDeclaratorNode]
1249
    #  in_pxd        boolean
Stefan Behnel's avatar
Stefan Behnel committed
1250
    #  api           boolean
1251
    #  overridable   boolean        whether it is a cpdef
1252
    #  modifiers     ['inline']
1253

1254
    #  decorators    [cython.locals(...)] or None
1255
    #  directive_locals { string : NameNode } locals defined by cython.locals(...)
1256 1257

    child_attrs = ["base_type", "declarators"]
1258

Robert Bradshaw's avatar
Robert Bradshaw committed
1259
    decorators = None
1260
    directive_locals = None
1261

William Stein's avatar
William Stein committed
1262
    def analyse_declarations(self, env, dest_scope = None):
1263 1264
        if self.directive_locals is None:
            self.directive_locals = {}
William Stein's avatar
William Stein committed
1265 1266
        if not dest_scope:
            dest_scope = env
1267
        self.dest_scope = dest_scope
1268 1269 1270 1271 1272 1273 1274 1275 1276 1277 1278 1279 1280 1281 1282

        if self.declarators:
            templates = self.declarators[0].analyse_templates()
        else:
            templates = None
        if templates is not None:
            if self.visibility != 'extern':
                error(self.pos, "Only extern functions allowed")
            if len(self.declarators) > 1:
                error(self.declarators[1].pos, "Can't multiply declare template types")
            env = TemplateScope('func_template', env)
            env.directives = env.outer_scope.directives
            for template_param in templates:
                env.declare_type(template_param.name, template_param, self.pos)

William Stein's avatar
William Stein committed
1283
        base_type = self.base_type.analyse(env)
1284

1285 1286
        if base_type.is_fused and not self.in_pxd and (env.is_c_class_scope or
                                                       env.is_module_scope):
1287 1288 1289
            error(self.pos, "Fused types not allowed here")
            return error_type

1290
        self.entry = None
1291
        visibility = self.visibility
1292

William Stein's avatar
William Stein committed
1293
        for declarator in self.declarators:
1294

1295 1296 1297
            if (len(self.declarators) > 1
                and not isinstance(declarator, CNameDeclaratorNode)
                and env.directives['warn.multiple_declarators']):
1298 1299 1300
                warning(declarator.pos,
                    "Non-trivial type declarators in shared declaration (e.g. mix of pointers and values). " +
                    "Each pointer declaration should be on its own line.", 1)
1301

1302 1303 1304 1305 1306
            create_extern_wrapper = (self.overridable
                                        and self.visibility == 'extern'
                                        and env.is_module_scope)
            if create_extern_wrapper:
                declarator.overridable = False
1307 1308 1309 1310
            if isinstance(declarator, CFuncDeclaratorNode):
                name_declarator, type = declarator.analyse(base_type, env, directive_locals=self.directive_locals)
            else:
                name_declarator, type = declarator.analyse(base_type, env)
William Stein's avatar
William Stein committed
1311
            if not type.is_complete():
1312
                if not (self.visibility == 'extern' and type.is_array or type.is_memoryviewslice):
William Stein's avatar
William Stein committed
1313 1314 1315 1316 1317 1318 1319
                    error(declarator.pos,
                        "Variable type '%s' is incomplete" % type)
            if self.visibility == 'extern' and type.is_pyobject:
                error(declarator.pos,
                    "Python object cannot be declared extern")
            name = name_declarator.name
            cname = name_declarator.cname
1320 1321 1322
            if name == '':
                error(declarator.pos, "Missing name in declaration.")
                return
William Stein's avatar
William Stein committed
1323
            if type.is_cfunction:
1324
                self.entry = dest_scope.declare_cfunction(name, type, declarator.pos,
1325 1326
                    cname=cname, visibility=self.visibility, in_pxd=self.in_pxd,
                    api=self.api, modifiers=self.modifiers, overridable=self.overridable)
1327 1328
                if self.entry is not None:
                    self.entry.directive_locals = copy.copy(self.directive_locals)
1329 1330
                if 'staticmethod' in env.directives:
                    type.is_static_method = True
1331
                if create_extern_wrapper:
1332
                    self.entry.type.create_to_py_utility_code(env)
1333
                    self.entry.create_wrapper = True
William Stein's avatar
William Stein committed
1334
            else:
1335
                if self.directive_locals:
1336
                    error(self.pos, "Decorators can only be followed by functions")
1337
                self.entry = dest_scope.declare_var(name, type, declarator.pos,
1338 1339
                            cname=cname, visibility=visibility, in_pxd=self.in_pxd,
                            api=self.api, is_cdef=1)
1340 1341
                if Options.docstrings:
                    self.entry.doc = embed_position(self.pos, self.doc)
1342

William Stein's avatar
William Stein committed
1343 1344 1345 1346 1347 1348

class CStructOrUnionDefNode(StatNode):
    #  name          string
    #  cname         string or None
    #  kind          "struct" or "union"
    #  typedef_flag  boolean
1349
    #  visibility    "public" or "private"
1350
    #  api           boolean
Stefan Behnel's avatar
Stefan Behnel committed
1351
    #  in_pxd        boolean
William Stein's avatar
William Stein committed
1352 1353
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1354
    #  packed        boolean
1355

1356
    child_attrs = ["attributes"]
Vitja Makarov's avatar
Vitja Makarov committed
1357

1358
    def declare(self, env, scope=None):
William Stein's avatar
William Stein committed
1359 1360
        self.entry = env.declare_struct_or_union(
            self.name, self.kind, scope, self.typedef_flag, self.pos,
1361 1362
            self.cname, visibility = self.visibility, api = self.api,
            packed = self.packed)
1363 1364 1365 1366 1367 1368

    def analyse_declarations(self, env):
        scope = None
        if self.attributes is not None:
            scope = StructOrUnionScope(self.name)
        self.declare(env, scope)
William Stein's avatar
William Stein committed
1369
        if self.attributes is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1370 1371
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
William Stein's avatar
William Stein committed
1372 1373
            for attr in self.attributes:
                attr.analyse_declarations(env, scope)
1374 1375 1376 1377
            if self.visibility != 'extern':
                for attr in scope.var_entries:
                    type = attr.type
                    while type.is_array:
1378 1379
                        type = type.base_type
                    if type == self.entry.type:
1380
                        error(attr.pos, "Struct cannot contain itself as a member.")
1381

William Stein's avatar
William Stein committed
1382
    def analyse_expressions(self, env):
1383
        return self
1384

William Stein's avatar
William Stein committed
1385 1386 1387 1388
    def generate_execution_code(self, code):
        pass


1389
class CppClassNode(CStructOrUnionDefNode, BlockNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
1390 1391 1392

    #  name          string
    #  cname         string or None
1393
    #  visibility    "extern"
Robert Bradshaw's avatar
Robert Bradshaw committed
1394 1395 1396
    #  in_pxd        boolean
    #  attributes    [CVarDefNode] or None
    #  entry         Entry
1397
    #  base_classes  [CBaseTypeNode]
Danilo Freitas's avatar
Danilo Freitas committed
1398
    #  templates     [string] or None
1399 1400 1401
    #  decorators    [DecoratorNode] or None

    decorators = None
Robert Bradshaw's avatar
Robert Bradshaw committed
1402

1403 1404 1405 1406 1407 1408 1409 1410 1411
    def declare(self, env):
        if self.templates is None:
            template_types = None
        else:
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name) for template_name in self.templates]
        self.entry = env.declare_cpp_class(
            self.name, None, self.pos,
            self.cname, base_classes = [], visibility = self.visibility, templates = template_types)

Robert Bradshaw's avatar
Robert Bradshaw committed
1412 1413
    def analyse_declarations(self, env):
        scope = None
1414
        if self.attributes is not None:
1415
            scope = CppClassScope(self.name, env, templates = self.templates)
1416 1417 1418 1419 1420 1421
        def base_ok(base_class):
            if base_class.is_cpp_class or base_class.is_struct:
                return True
            else:
                error(self.pos, "Base class '%s' not a struct or class." % base_class)
        base_class_types = filter(base_ok, [b.analyse(scope or env) for b in self.base_classes])
1422 1423 1424 1425
        if self.templates is None:
            template_types = None
        else:
            template_types = [PyrexTypes.TemplatePlaceholderType(template_name) for template_name in self.templates]
1426
        self.entry = env.declare_cpp_class(
1427
            self.name, scope, self.pos,
1428
            self.cname, base_class_types, visibility = self.visibility, templates = template_types)
1429 1430
        if self.entry is None:
            return
Danilo Freitas's avatar
Danilo Freitas committed
1431
        self.entry.is_cpp_class = 1
Stefan Behnel's avatar
Stefan Behnel committed
1432 1433
        if scope is not None:
            scope.type = self.entry.type
1434
        defined_funcs = []
1435 1436 1437 1438 1439 1440 1441
        def func_attributes(attributes):
            for attr in attributes:
                if isinstance(attr, CFuncDefNode):
                    yield attr
                elif isinstance(attr, CompilerDirectivesNode):
                    for sub_attr in func_attributes(attr.body.stats):
                        yield sub_attr
Robert Bradshaw's avatar
Robert Bradshaw committed
1442 1443 1444 1445
        if self.attributes is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for attr in self.attributes:
Robert Bradshaw's avatar
Robert Bradshaw committed
1446
                attr.analyse_declarations(scope)
1447 1448 1449 1450
            for func in func_attributes(self.attributes):
                defined_funcs.append(func)
                if self.templates is not None:
                    func.template_declaration = "template <typename %s>" % ", typename ".join(self.templates)
1451 1452 1453 1454
        self.body = StatListNode(self.pos, stats=defined_funcs)
        self.scope = scope

    def analyse_expressions(self, env):
1455 1456
        self.body = self.body.analyse_expressions(self.entry.type.scope)
        return self
1457 1458 1459 1460 1461 1462 1463 1464 1465 1466

    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(self.entry.type.scope, code)

    def generate_execution_code(self, code):
        self.body.generate_execution_code(code)

    def annotate(self, code):
        self.body.annotate(code)

Robert Bradshaw's avatar
Robert Bradshaw committed
1467

William Stein's avatar
William Stein committed
1468 1469 1470 1471 1472
class CEnumDefNode(StatNode):
    #  name           string or None
    #  cname          string or None
    #  items          [CEnumDefItemNode]
    #  typedef_flag   boolean
1473
    #  visibility     "public" or "private" or "extern"
1474
    #  api            boolean
Stefan Behnel's avatar
Stefan Behnel committed
1475
    #  in_pxd         boolean
1476
    #  create_wrapper boolean
William Stein's avatar
William Stein committed
1477
    #  entry          Entry
Robert Bradshaw's avatar
Robert Bradshaw committed
1478

1479
    child_attrs = ["items"]
1480

1481 1482 1483
    def declare(self, env):
         self.entry = env.declare_enum(self.name, self.pos,
             cname = self.cname, typedef_flag = self.typedef_flag,
1484 1485
             visibility = self.visibility, api = self.api,
             create_wrapper = self.create_wrapper)
1486

William Stein's avatar
William Stein committed
1487
    def analyse_declarations(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
1488 1489 1490 1491 1492
        if self.items is not None:
            if self.in_pxd and not env.in_cinclude:
                self.entry.defined_in_pxd = 1
            for item in self.items:
                item.analyse_declarations(env, self.entry)
William Stein's avatar
William Stein committed
1493

1494
    def analyse_expressions(self, env):
1495
        return self
1496

William Stein's avatar
William Stein committed
1497
    def generate_execution_code(self, code):
1498
        if self.visibility == 'public' or self.api:
1499
            temp = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
1500 1501
            for item in self.entry.enum_values:
                code.putln("%s = PyInt_FromLong(%s); %s" % (
1502
                        temp,
Robert Bradshaw's avatar
Robert Bradshaw committed
1503
                        item.cname,
1504
                        code.error_goto_if_null(temp, item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1505
                code.put_gotref(temp)
1506 1507
                code.putln('if (PyDict_SetItemString(%s, "%s", %s) < 0) %s' % (
                        Naming.moddict_cname,
1508
                        item.name,
1509
                        temp,
Robert Bradshaw's avatar
Robert Bradshaw committed
1510
                        code.error_goto(item.pos)))
Stefan Behnel's avatar
merge  
Stefan Behnel committed
1511
                code.put_decref_clear(temp, PyrexTypes.py_object_type)
1512
            code.funcstate.release_temp(temp)
William Stein's avatar
William Stein committed
1513 1514 1515 1516 1517 1518


class CEnumDefItemNode(StatNode):
    #  name     string
    #  cname    string or None
    #  value    ExprNode or None
1519

1520 1521
    child_attrs = ["value"]

William Stein's avatar
William Stein committed
1522 1523
    def analyse_declarations(self, env, enum_entry):
        if self.value:
1524
            self.value = self.value.analyse_const_expression(env)
1525 1526
            if not self.value.type.is_int:
                self.value = self.value.coerce_to(PyrexTypes.c_int_type, env)
1527
                self.value = self.value.analyse_const_expression(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
1528
        entry = env.declare_const(self.name, enum_entry.type,
1529
            self.value, self.pos, cname = self.cname,
1530 1531
            visibility = enum_entry.visibility, api = enum_entry.api,
            create_wrapper = enum_entry.create_wrapper)
William Stein's avatar
William Stein committed
1532 1533 1534 1535
        enum_entry.enum_values.append(entry)


class CTypeDefNode(StatNode):
Stefan Behnel's avatar
Stefan Behnel committed
1536 1537 1538
    #  base_type    CBaseTypeNode
    #  declarator   CDeclaratorNode
    #  visibility   "public" or "private"
1539
    #  api          boolean
Stefan Behnel's avatar
Stefan Behnel committed
1540
    #  in_pxd       boolean
1541 1542

    child_attrs = ["base_type", "declarator"]
1543

William Stein's avatar
William Stein committed
1544 1545 1546 1547 1548
    def analyse_declarations(self, env):
        base = self.base_type.analyse(env)
        name_declarator, type = self.declarator.analyse(base, env)
        name = name_declarator.name
        cname = name_declarator.cname
1549

Stefan Behnel's avatar
Stefan Behnel committed
1550
        entry = env.declare_typedef(name, type, self.pos,
1551
            cname = cname, visibility = self.visibility, api = self.api)
1552

Mark Florisson's avatar
Mark Florisson committed
1553
        if type.is_fused:
1554 1555
            entry.in_cinclude = True

Mark Florisson's avatar
Mark Florisson committed
1556 1557
        if self.in_pxd and not env.in_cinclude:
            entry.defined_in_pxd = 1
1558

William Stein's avatar
William Stein committed
1559
    def analyse_expressions(self, env):
1560 1561
        return self

William Stein's avatar
William Stein committed
1562 1563 1564 1565 1566 1567 1568 1569 1570 1571
    def generate_execution_code(self, code):
        pass


class FuncDefNode(StatNode, BlockNode):
    #  Base class for function definition nodes.
    #
    #  return_type     PyrexType
    #  #filename        string        C name of filename string const
    #  entry           Symtab.Entry
1572
    #  needs_closure   boolean        Whether or not this function has inner functions/classes/yield
Vitja Makarov's avatar
Vitja Makarov committed
1573
    #  needs_outer_scope boolean      Whether or not this function requires outer scope
1574
    #  pymethdef_required boolean     Force Python method struct generation
1575 1576
    #  directive_locals { string : ExprNode } locals defined by cython.locals(...)
    #  directive_returns [ExprNode] type defined by cython.returns(...)
1577 1578
    # star_arg      PyArgDeclNode or None  * argument
    # starstar_arg  PyArgDeclNode or None  ** argument
1579

1580 1581 1582 1583 1584
    #  has_fused_arguments  boolean
    #       Whether this cdef function has fused parameters. This is needed
    #       by AnalyseDeclarationsTransform, so it can replace CFuncDefNodes
    #       with fused argument types with a FusedCFuncDefNode

1585
    py_func = None
1586
    needs_closure = False
Vitja Makarov's avatar
Vitja Makarov committed
1587
    needs_outer_scope = False
1588
    pymethdef_required = False
1589
    is_generator = False
1590
    is_generator_body = False
Robert Bradshaw's avatar
Robert Bradshaw committed
1591
    modifiers = []
1592
    has_fused_arguments = False
1593 1594
    star_arg = None
    starstar_arg = None
1595
    is_cyfunction = False
1596
    code_object = None
1597

1598
    def analyse_default_values(self, env):
1599
        default_seen = 0
1600 1601
        for arg in self.args:
            if arg.default:
1602
                default_seen = 1
1603
                if arg.is_generic:
1604
                    arg.default = arg.default.analyse_types(env)
1605
                    arg.default = arg.default.coerce_to(arg.type, env)
1606 1607 1608 1609
                else:
                    error(arg.pos,
                        "This argument cannot have a default value")
                    arg.default = None
1610 1611 1612 1613
            elif arg.kw_only:
                default_seen = 1
            elif default_seen:
                error(arg.pos, "Non-default argument following default argument")
1614

1615 1616 1617 1618 1619
    def analyse_annotations(self, env):
        for arg in self.args:
            if arg.annotation:
                arg.annotation = arg.annotation.analyse_types(env)

1620
    def align_argument_type(self, env, arg):
1621
        # @cython.locals()
1622
        directive_locals = self.directive_locals
1623
        orig_type = arg.type
1624 1625 1626
        if arg.name in directive_locals:
            type_node = directive_locals[arg.name]
            other_type = type_node.analyse_as_type(env)
1627
        elif isinstance(arg, CArgDeclNode) and arg.annotation and env.directives['annotation_typing']:
1628 1629
            type_node = arg.annotation
            other_type = arg.inject_type_from_annotations(env)
1630 1631
            if other_type is None:
                return arg
1632 1633 1634 1635 1636 1637 1638 1639 1640 1641
        else:
            return arg
        if other_type is None:
            error(type_node.pos, "Not a type")
        elif (orig_type is not PyrexTypes.py_object_type
                and not orig_type.same_as(other_type)):
            error(arg.base_type.pos, "Signature does not agree with previous declaration")
            error(type_node.pos, "Previous declaration here")
        else:
            arg.type = other_type
1642 1643
        return arg

1644 1645
    def need_gil_acquisition(self, lenv):
        return 0
1646

1647 1648
    def create_local_scope(self, env):
        genv = env
1649
        while genv.is_py_class_scope or genv.is_c_class_scope:
1650
            genv = genv.outer_scope
1651
        if self.needs_closure:
1652 1653
            lenv = ClosureScope(name=self.entry.name,
                                outer_scope = genv,
1654
                                parent_scope = env,
1655
                                scope_name=self.entry.cname)
1656
        else:
1657 1658 1659
            lenv = LocalScope(name=self.entry.name,
                              outer_scope=genv,
                              parent_scope=env)
William Stein's avatar
William Stein committed
1660
        lenv.return_type = self.return_type
1661 1662 1663
        type = self.entry.type
        if type.is_cfunction:
            lenv.nogil = type.nogil and not type.with_gil
1664
        self.local_scope = lenv
1665
        lenv.directives = env.directives
1666
        return lenv
1667

1668 1669 1670
    def generate_function_body(self, env, code):
        self.body.generate_execution_code(code)

1671
    def generate_function_definitions(self, env, code):
1672
        from . import Buffer
1673
        if self.return_type.is_memoryviewslice:
1674
            from . import MemoryView
1675 1676

        lenv = self.local_scope
Vitja Makarov's avatar
Vitja Makarov committed
1677
        if lenv.is_closure_scope and not lenv.is_passthrough:
1678 1679 1680 1681 1682
            outer_scope_cname = "%s->%s" % (Naming.cur_scope_cname,
                                            Naming.outer_scope_cname)
        else:
            outer_scope_cname = Naming.outer_scope_cname
        lenv.mangle_closure_cnames(outer_scope_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
1683 1684
        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)
Stefan Behnel's avatar
Stefan Behnel committed
1685
        # generate lambda function definitions
1686
        self.generate_lambda_definitions(lenv, code)
1687

1688 1689
        is_getbuffer_slot = (self.entry.name == "__getbuffer__" and
                             self.entry.scope.is_c_class_scope)
1690 1691 1692
        is_releasebuffer_slot = (self.entry.name == "__releasebuffer__" and
                                 self.entry.scope.is_c_class_scope)
        is_buffer_slot = is_getbuffer_slot or is_releasebuffer_slot
1693
        if is_buffer_slot:
1694 1695
            if 'cython_unused' not in self.modifiers:
                self.modifiers = self.modifiers + ['cython_unused']
1696

1697
        preprocessor_guard = self.get_preprocessor_guard()
1698

1699
        profile = code.globalstate.directives['profile']
1700 1701
        linetrace = code.globalstate.directives['linetrace']
        if (linetrace or profile) and lenv.nogil:
1702
            warning(self.pos, "Cannot profile nogil function.", 1)
1703 1704
            profile = linetrace = False
        if profile or linetrace:
1705 1706
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("Profile", "Profile.c"))
1707

1708
        # Generate C code for header and body of function
1709
        code.enter_cfunc_scope()
1710
        code.return_from_error_cleanup_label = code.new_label()
1711

William Stein's avatar
William Stein committed
1712
        # ----- Top-level constants used by this function
1713
        code.mark_pos(self.pos)
1714
        self.generate_cached_builtins_decls(lenv, code)
William Stein's avatar
William Stein committed
1715 1716
        # ----- Function header
        code.putln("")
1717 1718 1719 1720

        if preprocessor_guard:
            code.putln(preprocessor_guard)

1721 1722
        with_pymethdef = (self.needs_assignment_synthesis(env, code) or
                          self.pymethdef_required)
1723
        if self.py_func:
1724
            self.py_func.generate_function_header(code,
Stefan Behnel's avatar
Stefan Behnel committed
1725
                with_pymethdef = with_pymethdef,
1726
                proto_only=True)
William Stein's avatar
William Stein committed
1727
        self.generate_function_header(code,
Stefan Behnel's avatar
Stefan Behnel committed
1728
            with_pymethdef = with_pymethdef)
William Stein's avatar
William Stein committed
1729
        # ----- Local variable declarations
1730 1731 1732 1733
        # Find function scope
        cenv = env
        while cenv.is_py_class_scope or cenv.is_c_class_scope:
            cenv = cenv.outer_scope
Vitja Makarov's avatar
Vitja Makarov committed
1734
        if self.needs_closure:
1735
            code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
Robert Bradshaw's avatar
Robert Bradshaw committed
1736
            code.putln(";")
Vitja Makarov's avatar
Vitja Makarov committed
1737 1738 1739 1740
        elif self.needs_outer_scope:
            if lenv.is_passthrough:
                code.put(lenv.scope_class.type.declaration_code(Naming.cur_scope_cname))
                code.putln(";")
1741
            code.put(cenv.scope_class.type.declaration_code(Naming.outer_scope_cname))
1742
            code.putln(";")
William Stein's avatar
William Stein committed
1743
        self.generate_argument_declarations(lenv, code)
1744

1745
        for entry in lenv.var_entries:
1746
            if not (entry.in_closure or entry.is_arg):
1747
                code.put_var_declaration(entry)
1748

1749
        # Initialize the return variable __pyx_r
William Stein's avatar
William Stein committed
1750 1751
        init = ""
        if not self.return_type.is_void:
1752 1753
            if self.return_type.is_pyobject:
                init = " = NULL"
1754
            elif self.return_type.is_memoryviewslice:
1755
                init = ' = ' + MemoryView.memslice_entry_init
1756

William Stein's avatar
William Stein committed
1757
            code.putln(
1758
                "%s%s;" %
1759 1760
                    (self.return_type.declaration_code(Naming.retval_cname),
                     init))
1761

1762
        tempvardecl_code = code.insertion_point()
William Stein's avatar
William Stein committed
1763
        self.generate_keyword_list(code)
1764

1765
        if profile or linetrace:
1766 1767
            code_object = self.code_object.calculate_result_code(code) if self.code_object else None
            code.put_trace_declarations(code_object)
1768

William Stein's avatar
William Stein committed
1769 1770
        # ----- Extern library function declarations
        lenv.generate_library_function_declarations(code)
1771

1772
        # ----- GIL acquisition
1773
        acquire_gil = self.acquire_gil
1774

Mark Florisson's avatar
Mark Florisson committed
1775 1776
        # See if we need to acquire the GIL for variable declarations, or for
        # refnanny only
1777

Mark Florisson's avatar
Mark Florisson committed
1778 1779 1780
        # Profiling or closures are not currently possible for cdef nogil
        # functions, but check them anyway
        have_object_args = (self.needs_closure or self.needs_outer_scope or
1781
                            profile or linetrace)
Mark Florisson's avatar
Mark Florisson committed
1782 1783 1784 1785 1786 1787 1788 1789 1790 1791 1792 1793 1794 1795
        for arg in lenv.arg_entries:
            if arg.type.is_pyobject:
                have_object_args = True
                break

        acquire_gil_for_var_decls_only = (
                lenv.nogil and lenv.has_with_gil_block and
                (have_object_args or lenv.buffer_entries))

        acquire_gil_for_refnanny_only = (
                lenv.nogil and lenv.has_with_gil_block and not
                acquire_gil_for_var_decls_only)

        use_refnanny = not lenv.nogil or lenv.has_with_gil_block
1796 1797 1798

        if acquire_gil or acquire_gil_for_var_decls_only:
            code.put_ensure_gil()
1799 1800
        elif lenv.nogil and lenv.has_with_gil_block:
            code.declare_gilstate()
1801

1802
        # ----- set up refnanny
1803
        if use_refnanny:
1804
            tempvardecl_code.put_declare_refcount_context()
1805 1806
            code.put_setup_refcount_context(
                self.entry.name, acquire_gil=acquire_gil_for_refnanny_only)
Mark Florisson's avatar
Mark Florisson committed
1807

1808
        # ----- Automatic lead-ins for certain special functions
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1809 1810
        if is_getbuffer_slot:
            self.getbuffer_init(code)
1811
        # ----- Create closure scope object
Robert Bradshaw's avatar
Robert Bradshaw committed
1812
        if self.needs_closure:
1813
            tp_slot = TypeSlots.ConstructorSlot("tp_new", '__new__')
1814 1815 1816 1817
            slot_func_cname = TypeSlots.get_slot_function(lenv.scope_class.type.scope, tp_slot)
            if not slot_func_cname:
                slot_func_cname = '%s->tp_new' % lenv.scope_class.type.typeptr_cname
            code.putln("%s = (%s)%s(%s, %s, NULL);" % (
1818
                Naming.cur_scope_cname,
1819
                lenv.scope_class.type.empty_declaration_code(),
1820
                slot_func_cname,
1821
                lenv.scope_class.type.typeptr_cname,
1822 1823 1824 1825
                Naming.empty_tuple))
            code.putln("if (unlikely(!%s)) {" % Naming.cur_scope_cname)
            if is_getbuffer_slot:
                self.getbuffer_error_cleanup(code)
1826 1827

            if use_refnanny:
1828
                code.put_finish_refcount_context()
Mark Florisson's avatar
Mark Florisson committed
1829
                if acquire_gil or acquire_gil_for_var_decls_only:
1830 1831
                    code.put_release_ensured_gil()

1832 1833 1834
            # FIXME: what if the error return value is a Python value?
            code.putln("return %s;" % self.error_value())
            code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
1835
            code.put_gotref(Naming.cur_scope_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
1836
            # Note that it is unsafe to decref the scope at this point.
Vitja Makarov's avatar
Vitja Makarov committed
1837
        if self.needs_outer_scope:
1838 1839 1840
            if self.is_cyfunction:
                code.putln("%s = (%s) __Pyx_CyFunction_GetClosure(%s);" % (
                    outer_scope_cname,
1841
                    cenv.scope_class.type.empty_declaration_code(),
1842 1843 1844 1845
                    Naming.self_cname))
            else:
                code.putln("%s = (%s) %s;" % (
                    outer_scope_cname,
1846
                    cenv.scope_class.type.empty_declaration_code(),
1847
                    Naming.self_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1848
            if lenv.is_passthrough:
Stefan Behnel's avatar
Stefan Behnel committed
1849
                code.putln("%s = %s;" % (Naming.cur_scope_cname, outer_scope_cname))
Vitja Makarov's avatar
Vitja Makarov committed
1850
            elif self.needs_closure:
1851
                # inner closures own a reference to their outer parent
1852
                code.put_incref(outer_scope_cname, cenv.scope_class.type)
1853
                code.put_giveref(outer_scope_cname)
1854
        # ----- Trace function call
1855
        if profile or linetrace:
1856 1857 1858
            # this looks a bit late, but if we don't get here due to a
            # fatal error before hand, it's not really worth tracing
            code.put_trace_call(self.entry.name, self.pos)
1859
            code.funcstate.can_trace = True
William Stein's avatar
William Stein committed
1860
        # ----- Fetch arguments
1861
        self.generate_argument_parsing_code(env, code)
1862
        # If an argument is assigned to in the body, we must
Robert Bradshaw's avatar
Robert Bradshaw committed
1863
        # incref it to properly keep track of refcounts.
1864
        is_cdef = isinstance(self, CFuncDefNode)
Robert Bradshaw's avatar
Robert Bradshaw committed
1865
        for entry in lenv.arg_entries:
1866
            if entry.type.is_pyobject:
1867 1868
                if ((acquire_gil or len(entry.cf_assignments) > 1) and
                    not entry.in_closure):
1869
                    code.put_var_incref(entry)
1870

1871
            # Note: defaults are always incref-ed. For def functions, we
1872 1873 1874
            #       we aquire arguments from object converstion, so we have
            #       new references. If we are a cdef function, we need to
            #       incref our arguments
Mark Florisson's avatar
Mark Florisson committed
1875 1876
            elif (is_cdef and entry.type.is_memoryviewslice and
                  len(entry.cf_assignments) > 1):
1877 1878
                code.put_incref_memoryviewslice(entry.cname,
                                                have_gil=not lenv.nogil)
1879 1880 1881
        for entry in lenv.var_entries:
            if entry.is_arg and len(entry.cf_assignments) > 1:
                code.put_var_incref(entry)
1882

1883 1884
        # ----- Initialise local buffer auxiliary variables
        for entry in lenv.var_entries + lenv.arg_entries:
1885 1886
            if entry.type.is_buffer and entry.buffer_aux.buflocal_nd_var.used:
                Buffer.put_init_vars(entry, code)
1887

1888
        # ----- Check and convert arguments
William Stein's avatar
William Stein committed
1889
        self.generate_argument_type_tests(code)
1890 1891 1892
        # ----- Acquire buffer arguments
        for entry in lenv.arg_entries:
            if entry.type.is_buffer:
1893 1894
                Buffer.put_acquire_arg_buffer(entry, code, self.pos)

1895 1896 1897
        if acquire_gil_for_var_decls_only:
            code.put_release_ensured_gil()

1898 1899 1900
        # -------------------------
        # ----- Function body -----
        # -------------------------
1901
        self.generate_function_body(env, code)
1902

1903
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
1904
        code.putln("")
1905 1906 1907
        code.putln("/* function exit code */")

        # ----- Default return value
1908 1909 1910 1911 1912 1913 1914 1915 1916 1917 1918
        if not self.body.is_terminator:
            if self.return_type.is_pyobject:
                #if self.return_type.is_extension_type:
                #    lhs = "(PyObject *)%s" % Naming.retval_cname
                #else:
                lhs = Naming.retval_cname
                code.put_init_to_py_none(lhs, self.return_type)
            else:
                val = self.return_type.default_value
                if val:
                    code.putln("%s = %s;" % (Naming.retval_cname, val))
William Stein's avatar
William Stein committed
1919
        # ----- Error cleanup
1920
        if code.error_label in code.labels_used:
1921 1922
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
1923
            code.put_label(code.error_label)
1924
            for cname, type in code.funcstate.all_managed_temps():
1925
                code.put_xdecref(cname, type, have_gil=not lenv.nogil)
1926 1927 1928 1929

            # Clean up buffers -- this calls a Python function
            # so need to save and restore error state
            buffers_present = len(lenv.buffer_entries) > 0
1930 1931
            memslice_entries = [e for e in lenv.entries.itervalues()
                                      if e.type.is_memoryviewslice]
1932
            if buffers_present:
1933
                code.globalstate.use_utility_code(restore_exception_utility_code)
1934
                code.putln("{ PyObject *__pyx_type, *__pyx_value, *__pyx_tb;")
1935
                code.putln("__Pyx_ErrFetch(&__pyx_type, &__pyx_value, &__pyx_tb);")
1936
                for entry in lenv.buffer_entries:
1937
                    Buffer.put_release_buffer_code(code, entry)
1938
                    #code.putln("%s = 0;" % entry.cname)
1939
                code.putln("__Pyx_ErrRestore(__pyx_type, __pyx_value, __pyx_tb);}")
1940

1941 1942 1943 1944 1945 1946
            if self.return_type.is_memoryviewslice:
                MemoryView.put_init_entry(Naming.retval_cname, code)
                err_val = Naming.retval_cname
            else:
                err_val = self.error_value()

1947 1948
            exc_check = self.caller_will_check_exceptions()
            if err_val is not None or exc_check:
1949
                # TODO: Fix exception tracing (though currently unused by cProfile).
Robert Bradshaw's avatar
Robert Bradshaw committed
1950 1951
                # code.globalstate.use_utility_code(get_exception_tuple_utility_code)
                # code.put_trace_exception()
1952

Mark Florisson's avatar
Mark Florisson committed
1953
                if lenv.nogil and not lenv.has_with_gil_block:
1954 1955 1956
                    code.putln("{")
                    code.put_ensure_gil()

1957
                code.put_add_traceback(self.entry.qualified_name)
1958

Mark Florisson's avatar
Mark Florisson committed
1959
                if lenv.nogil and not lenv.has_with_gil_block:
1960 1961
                    code.put_release_ensured_gil()
                    code.putln("}")
1962
            else:
1963 1964 1965 1966
                warning(self.entry.pos,
                        "Unraisable exception in function '%s'." %
                        self.entry.qualified_name, 0)
                code.put_unraisable(self.entry.qualified_name)
1967 1968 1969 1970
            default_retval = self.return_type.default_value
            if err_val is None and default_retval:
                err_val = default_retval
            if err_val is not None:
1971
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
1972 1973 1974 1975 1976 1977 1978

            if is_getbuffer_slot:
                self.getbuffer_error_cleanup(code)

            # If we are using the non-error cleanup section we should
            # jump past it if we have an error. The if-test below determine
            # whether this section is used.
1979
            if buffers_present or is_getbuffer_slot or self.return_type.is_memoryviewslice:
1980 1981 1982
                code.put_goto(code.return_from_error_cleanup_label)

        # ----- Non-error return cleanup
William Stein's avatar
William Stein committed
1983
        code.put_label(code.return_label)
1984
        for entry in lenv.buffer_entries:
1985
            if entry.used:
1986
                Buffer.put_release_buffer_code(code, entry)
1987 1988
        if is_getbuffer_slot:
            self.getbuffer_normal_cleanup(code)
1989 1990 1991

        if self.return_type.is_memoryviewslice:
            # See if our return value is uninitialized on non-error return
1992
            # from . import MemoryView
1993 1994 1995 1996 1997 1998 1999 2000 2001 2002 2003 2004 2005 2006 2007 2008
            # MemoryView.err_if_nogil_initialized_check(self.pos, env)
            cond = code.unlikely(self.return_type.error_condition(
                                                    Naming.retval_cname))
            code.putln(
                'if (%s) {' % cond)
            if env.nogil:
                code.put_ensure_gil()
            code.putln(
                    'PyErr_SetString('
                        'PyExc_TypeError,'
                        '"Memoryview return value is not initialized");')
            if env.nogil:
                code.put_release_ensured_gil()
            code.putln(
                '}')

2009 2010
        # ----- Return cleanup for both error and no-error return
        code.put_label(code.return_from_error_cleanup_label)
2011

2012
        for entry in lenv.var_entries:
2013 2014
            if not entry.used or entry.in_closure:
                continue
2015

2016
            if entry.type.is_memoryviewslice:
2017 2018
                code.put_xdecref_memoryviewslice(entry.cname,
                                                 have_gil=not lenv.nogil)
2019
            elif entry.type.is_pyobject:
2020 2021
                if not entry.is_arg or len(entry.cf_assignments) > 1:
                    code.put_var_decref(entry)
2022

Robert Bradshaw's avatar
Robert Bradshaw committed
2023
        # Decref any increfed args
2024
        for entry in lenv.arg_entries:
2025
            if entry.type.is_pyobject:
2026 2027
                if ((acquire_gil or len(entry.cf_assignments) > 1) and
                    not entry.in_closure):
2028
                    code.put_var_decref(entry)
2029 2030 2031 2032
            elif (entry.type.is_memoryviewslice and
                  (not is_cdef or len(entry.cf_assignments) > 1)):
                # decref slices of def functions and acquired slices from cdef
                # functions, but not borrowed slices from cdef functions.
2033 2034
                code.put_xdecref_memoryviewslice(entry.cname,
                                                 have_gil=not lenv.nogil)
Robert Bradshaw's avatar
Robert Bradshaw committed
2035 2036
        if self.needs_closure:
            code.put_decref(Naming.cur_scope_cname, lenv.scope_class.type)
2037

2038
        # ----- Return
2039
        # This code is duplicated in ModuleNode.generate_module_init_func
2040 2041 2042 2043
        if not lenv.nogil:
            default_retval = self.return_type.default_value
            err_val = self.error_value()
            if err_val is None and default_retval:
Stefan Behnel's avatar
Stefan Behnel committed
2044
                err_val = default_retval  # FIXME: why is err_val not used?
2045 2046
            if self.return_type.is_pyobject:
                code.put_xgiveref(self.return_type.as_pyobject(Naming.retval_cname))
2047

2048 2049
        if self.entry.is_special and self.entry.name == "__hash__":
            # Returning -1 for __hash__ is supposed to signal an error
2050
            # We do as Python instances and coerce -1 into -2.
2051 2052
            code.putln("if (unlikely(%s == -1) && !PyErr_Occurred()) %s = -2;" % (
                    Naming.retval_cname, Naming.retval_cname))
2053

2054 2055
        if profile or linetrace:
            code.funcstate.can_trace = False
Robert Bradshaw's avatar
Robert Bradshaw committed
2056 2057 2058 2059
            if self.return_type.is_pyobject:
                code.put_trace_return(Naming.retval_cname)
            else:
                code.put_trace_return("Py_None")
2060

2061
        if not lenv.nogil:
Stefan Behnel's avatar
Stefan Behnel committed
2062
            # GIL holding function
2063
            code.put_finish_refcount_context()
2064

2065 2066
        if acquire_gil or (lenv.nogil and lenv.has_with_gil_block):
            # release the GIL (note that with-gil blocks acquire it on exit in their EnsureGILNode)
2067
            code.put_release_ensured_gil()
2068

William Stein's avatar
William Stein committed
2069
        if not self.return_type.is_void:
2070
            code.putln("return %s;" % Naming.retval_cname)
2071

William Stein's avatar
William Stein committed
2072
        code.putln("}")
2073 2074 2075 2076

        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

2077
        # ----- Go back and insert temp variable declarations
2078
        tempvardecl_code.put_temp_declarations(code.funcstate)
2079

2080
        # ----- Python version
2081
        code.exit_cfunc_scope()
2082
        if self.py_func:
2083
            self.py_func.generate_function_definitions(env, code)
2084
        self.generate_wrapper_functions(code)
William Stein's avatar
William Stein committed
2085 2086 2087 2088

    def declare_argument(self, env, arg):
        if arg.type.is_void:
            error(arg.pos, "Invalid use of 'void'")
2089
        elif not arg.type.is_complete() and not (arg.type.is_array or arg.type.is_memoryviewslice):
William Stein's avatar
William Stein committed
2090 2091 2092
            error(arg.pos,
                "Argument type '%s' is incomplete" % arg.type)
        return env.declare_arg(arg.name, arg.type, arg.pos)
2093

2094 2095 2096
    def generate_arg_type_test(self, arg, code):
        # Generate type test for one argument.
        if arg.type.typeobj_is_available():
2097 2098
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ArgTypeTest", "FunctionArguments.c"))
2099 2100 2101 2102
            typeptr_cname = arg.type.typeptr_cname
            arg_code = "((PyObject *)%s)" % arg.entry.cname
            code.putln(
                'if (unlikely(!__Pyx_ArgTypeTest(%s, %s, %d, "%s", %s))) %s' % (
2103
                    arg_code,
2104 2105 2106 2107 2108 2109 2110 2111
                    typeptr_cname,
                    arg.accept_none,
                    arg.name,
                    arg.type.is_builtin_type,
                    code.error_goto(arg.pos)))
        else:
            error(arg.pos, "Cannot test type of extern C class "
                "without type object name specification")
2112 2113 2114

    def generate_arg_none_check(self, arg, code):
        # Generate None check for one argument.
2115 2116 2117 2118 2119 2120
        if arg.type.is_memoryviewslice:
            cname = "%s.memview" % arg.entry.cname
        else:
            cname = arg.entry.cname

        code.putln('if (unlikely(((PyObject *)%s) == Py_None)) {' % cname)
2121 2122
        code.putln('''PyErr_Format(PyExc_TypeError, "Argument '%%.%ds' must not be None", "%s"); %s''' % (
            max(200, len(arg.name)), arg.name,
2123 2124
            code.error_goto(arg.pos)))
        code.putln('}')
2125

2126
    def generate_wrapper_functions(self, code):
William Stein's avatar
William Stein committed
2127 2128 2129
        pass

    def generate_execution_code(self, code):
2130 2131
        # Evaluate and store argument default values
        for arg in self.args:
2132 2133
            if not arg.is_dynamic:
                arg.generate_assignment_code(code)
William Stein's avatar
William Stein committed
2134

2135
    #
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2136
    # Special code for the __getbuffer__ function
2137 2138 2139 2140 2141
    #
    def getbuffer_init(self, code):
        info = self.local_scope.arg_entries[1].cname
        # Python 3.0 betas have a bug in memoryview which makes it call
        # getbuffer with a NULL parameter. For now we work around this;
2142 2143
        # the following block should be removed when this bug is fixed.
        code.putln("if (%s != NULL) {" % info)
2144
        code.putln("%s->obj = Py_None; __Pyx_INCREF(Py_None);" % info)
2145
        code.put_giveref("%s->obj" % info) # Do not refnanny object within structs
2146
        code.putln("}")
2147 2148 2149

    def getbuffer_error_cleanup(self, code):
        info = self.local_scope.arg_entries[1].cname
2150 2151
        code.putln("if (%s != NULL && %s->obj != NULL) {"
                   % (info, info))
2152
        code.put_gotref("%s->obj" % info)
2153 2154 2155
        code.putln("__Pyx_DECREF(%s->obj); %s->obj = NULL;"
                   % (info, info))
        code.putln("}")
2156 2157 2158

    def getbuffer_normal_cleanup(self, code):
        info = self.local_scope.arg_entries[1].cname
2159
        code.putln("if (%s != NULL && %s->obj == Py_None) {" % (info, info))
2160 2161 2162
        code.put_gotref("Py_None")
        code.putln("__Pyx_DECREF(Py_None); %s->obj = NULL;" % info)
        code.putln("}")
William Stein's avatar
William Stein committed
2163

2164
    def get_preprocessor_guard(self):
2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175
        if not self.entry.is_special:
            return None
        name = self.entry.name
        slot = TypeSlots.method_name_to_slot.get(name)
        if not slot:
            return None
        if name == '__long__' and not self.entry.scope.lookup_here('__int__'):
            return None
        if name in ("__getbuffer__", "__releasebuffer__") and self.entry.scope.is_c_class_scope:
            return None
        return slot.preprocessor_guard_code()
2176 2177


William Stein's avatar
William Stein committed
2178 2179 2180
class CFuncDefNode(FuncDefNode):
    #  C function definition.
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
2181
    #  modifiers     ['inline']
William Stein's avatar
William Stein committed
2182 2183 2184
    #  visibility    'private' or 'public' or 'extern'
    #  base_type     CBaseTypeNode
    #  declarator    CDeclaratorNode
2185 2186 2187
    #  cfunc_declarator  the CFuncDeclarator of this function
    #                    (this is also available through declarator or a
    #                     base thereof)
William Stein's avatar
William Stein committed
2188
    #  body          StatListNode
2189
    #  api           boolean
2190
    #  decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2191
    #
2192
    #  with_gil      boolean    Acquire GIL around body
William Stein's avatar
William Stein committed
2193
    #  type          CFuncType
2194
    #  py_func       wrapper for calling from Python
2195
    #  overridable   whether or not this is a cpdef function
2196
    #  inline_in_pxd whether this is an inline function in a pxd file
2197
    #  template_declaration  String or None   Used for c++ class methods
Robert Bradshaw's avatar
Robert Bradshaw committed
2198
    #  is_const_method whether this is a const method
Robert Bradshaw's avatar
Robert Bradshaw committed
2199
    #  is_static_method whether this is a static method
2200
    #  is_c_class_method whether this is a cclass method
2201

2202
    child_attrs = ["base_type", "declarator", "body", "py_func_stat"]
2203 2204

    inline_in_pxd = False
2205
    decorators = None
2206
    directive_locals = None
2207
    directive_returns = None
2208
    override = None
2209
    template_declaration = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2210
    is_const_method = False
2211
    py_func_stat = None
2212

William Stein's avatar
William Stein committed
2213 2214
    def unqualified_name(self):
        return self.entry.name
2215

William Stein's avatar
William Stein committed
2216
    def analyse_declarations(self, env):
2217
        self.is_c_class_method = env.is_c_class_scope
2218 2219
        if self.directive_locals is None:
            self.directive_locals = {}
2220
        self.directive_locals.update(env.directives['locals'])
2221 2222 2223 2224 2225 2226 2227
        if self.directive_returns is not None:
            base_type = self.directive_returns.analyse_as_type(env)
            if base_type is None:
                error(self.directive_returns.pos, "Not a type")
                base_type = PyrexTypes.error_type
        else:
            base_type = self.base_type.analyse(env)
2228
        self.is_static_method = 'staticmethod' in env.directives and not env.lookup_here('staticmethod')
2229
        # The 2 here is because we need both function and argument names.
2230 2231 2232 2233 2234 2235
        if isinstance(self.declarator, CFuncDeclaratorNode):
            name_declarator, type = self.declarator.analyse(base_type, env,
                                                            nonempty = 2 * (self.body is not None),
                                                            directive_locals = self.directive_locals)
        else:
            name_declarator, type = self.declarator.analyse(base_type, env, nonempty = 2 * (self.body is not None))
2236
        if not type.is_cfunction:
2237
            error(self.pos,
2238
                "Suite attached to non-function declaration")
William Stein's avatar
William Stein committed
2239 2240 2241 2242 2243
        # Remember the actual type according to the function header
        # written here, because the type in the symbol table entry
        # may be different if we're overriding a C method inherited
        # from the base type of an extension type.
        self.type = type
2244
        type.is_overridable = self.overridable
2245 2246 2247
        declarator = self.declarator
        while not hasattr(declarator, 'args'):
            declarator = declarator.base
2248 2249

        self.cfunc_declarator = declarator
2250
        self.args = declarator.args
2251

2252 2253 2254 2255 2256 2257
        opt_arg_count = self.cfunc_declarator.optional_arg_count
        if (self.visibility == 'public' or self.api) and opt_arg_count:
            error(self.cfunc_declarator.pos,
                  "Function with optional arguments may not be declared "
                  "public or api")

2258 2259 2260 2261
        if (type.exception_check == '+' and self.visibility != 'extern'):
            warning(self.cfunc_declarator.pos,
                    "Only extern functions can throw C++ exceptions.")

2262
        for formal_arg, type_arg in zip(self.args, type.args):
2263
            self.align_argument_type(env, type_arg)
2264
            formal_arg.type = type_arg.type
2265
            formal_arg.name = type_arg.name
2266
            formal_arg.cname = type_arg.cname
2267

2268 2269
            self._validate_type_visibility(type_arg.type, type_arg.pos, env)

2270 2271 2272
            if type_arg.type.is_fused:
                self.has_fused_arguments = True

2273 2274
            if type_arg.type.is_buffer and 'inline' in self.modifiers:
                warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)
2275

2276 2277 2278 2279 2280 2281 2282 2283
            if type_arg.type.is_buffer:
                if self.type.nogil:
                    error(formal_arg.pos,
                          "Buffer may not be acquired without the GIL. "
                          "Consider using memoryview slices instead.")
                elif 'inline' in self.modifiers:
                    warning(formal_arg.pos, "Buffer unpacking not optimized away.", 1)

2284 2285
        self._validate_type_visibility(type.return_type, self.pos, env)

William Stein's avatar
William Stein committed
2286 2287
        name = name_declarator.name
        cname = name_declarator.cname
2288

Robert Bradshaw's avatar
Robert Bradshaw committed
2289
        type.is_const_method = self.is_const_method
Robert Bradshaw's avatar
Robert Bradshaw committed
2290
        type.is_static_method = self.is_static_method
William Stein's avatar
William Stein committed
2291
        self.entry = env.declare_cfunction(
2292
            name, type, self.pos,
2293 2294 2295
            cname=cname, visibility=self.visibility, api=self.api,
            defining=self.body is not None, modifiers=self.modifiers,
            overridable=self.overridable)
2296
        self.entry.inline_func_in_pxd = self.inline_in_pxd
William Stein's avatar
William Stein committed
2297
        self.return_type = type.return_type
2298
        if self.return_type.is_array and self.visibility != 'extern':
2299 2300
            error(self.pos,
                "Function cannot return an array")
2301 2302
        if self.return_type.is_cpp_class:
            self.return_type.check_nullary_constructor(self.pos, "used as a return value")
2303

Robert Bradshaw's avatar
Robert Bradshaw committed
2304
        if self.overridable and not env.is_module_scope and not self.is_static_method:
2305 2306 2307
            if len(self.args) < 1 or not self.args[0].type.is_pyobject:
                # An error will be produced in the cdef function
                self.overridable = False
2308

2309 2310 2311 2312
        self.declare_cpdef_wrapper(env)
        self.create_local_scope(env)

    def declare_cpdef_wrapper(self, env):
2313
        if self.overridable:
2314 2315 2316
            if self.is_static_method:
                # TODO(robertwb): Finish this up, perhaps via more function refactoring.
                error(self.pos, "static cpdef methods not yet supported")
2317
            name = self.entry.name
2318
            py_func_body = self.call_self_node(is_module_scope = env.is_module_scope)
2319 2320 2321 2322 2323 2324
            if self.is_static_method:
                from .ExprNodes import NameNode
                decorators = [DecoratorNode(self.pos, decorator=NameNode(self.pos, name='staticmethod'))]
                decorators[0].decorator.analyse_types(env)
            else:
                decorators = []
2325
            self.py_func = DefNode(pos = self.pos,
2326 2327
                                   name = self.entry.name,
                                   args = self.args,
2328 2329
                                   star_arg = None,
                                   starstar_arg = None,
2330
                                   doc = self.doc,
2331
                                   body = py_func_body,
2332
                                   decorators = decorators,
2333
                                   is_wrapper = 1)
2334
            self.py_func.is_module_scope = env.is_module_scope
2335
            self.py_func.analyse_declarations(env)
2336 2337
            self.py_func_stat = StatListNode(pos = self.pos, stats = [self.py_func])
            self.py_func.type = PyrexTypes.py_object_type
2338
            self.entry.as_variable = self.py_func.entry
2339
            self.entry.used = self.entry.as_variable.used = True
2340 2341
            # Reset scope entry the above cfunction
            env.entries[name] = self.entry
2342 2343
            if (not self.entry.is_final_cmethod and
                (not env.is_module_scope or Options.lookup_module_cpdef)):
2344 2345
                self.override = OverrideCheckNode(self.pos, py_func = self.py_func)
                self.body = StatListNode(self.pos, stats=[self.override, self.body])
2346

2347 2348 2349 2350 2351 2352 2353 2354 2355 2356 2357 2358 2359
    def _validate_type_visibility(self, type, pos, env):
        """
        Ensure that types used in cdef functions are public or api, or
        defined in a C header.
        """
        public_or_api = (self.visibility == 'public' or self.api)
        entry = getattr(type, 'entry', None)
        if public_or_api and entry and env.is_module_scope:
            if not (entry.visibility in ('public', 'extern') or
                    entry.api or entry.in_cinclude):
                error(pos, "Function declared public or api may not have "
                           "private types")

2360
    def call_self_node(self, omit_optional_args=0, is_module_scope=0):
2361
        from . import ExprNodes
2362 2363 2364 2365
        args = self.type.args
        if omit_optional_args:
            args = args[:len(args) - self.type.optional_arg_count]
        arg_names = [arg.name for arg in args]
2366
        if is_module_scope:
2367
            cfunc = ExprNodes.NameNode(self.pos, name=self.entry.name)
2368 2369 2370 2371
            call_arg_names = arg_names
            skip_dispatch = Options.lookup_module_cpdef
        elif self.type.is_static_method:
            class_entry = self.entry.scope.parent_type.entry
2372 2373 2374
            class_node = ExprNodes.NameNode(self.pos, name=class_entry.name)
            class_node.entry = class_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=class_node, attribute=self.entry.name)
2375 2376 2377
            # Calling static c(p)def methods on an instance disallowed.
            # TODO(robertwb): Support by passing self to check for override?
            skip_dispatch = True
2378
        else:
2379 2380 2381 2382
            type_entry = self.type.args[0].type.entry
            type_arg = ExprNodes.NameNode(self.pos, name=type_entry.name)
            type_arg.entry = type_entry
            cfunc = ExprNodes.AttributeNode(self.pos, obj=type_arg, attribute=self.entry.name)
2383
        skip_dispatch = not is_module_scope or Options.lookup_module_cpdef
2384 2385 2386 2387 2388
        c_call = ExprNodes.SimpleCallNode(
            self.pos,
            function=cfunc,
            args=[ExprNodes.NameNode(self.pos, name=n) for n in arg_names],
            wrapper_call=skip_dispatch)
2389 2390
        return ReturnStatNode(pos=self.pos, return_type=PyrexTypes.py_object_type, value=c_call)

William Stein's avatar
William Stein committed
2391 2392 2393 2394 2395
    def declare_arguments(self, env):
        for arg in self.type.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            self.declare_argument(env, arg)
2396

2397
    def need_gil_acquisition(self, lenv):
2398 2399
        return self.type.with_gil

2400
    def nogil_check(self, env):
2401
        type = self.type
2402
        with_gil = type.with_gil
2403 2404 2405 2406
        if type.nogil and not with_gil:
            if type.return_type.is_pyobject:
                error(self.pos,
                      "Function with Python return type cannot be declared nogil")
2407
            for entry in self.local_scope.var_entries:
2408
                if entry.type.is_pyobject and not entry.in_with_gil_block:
2409 2410
                    error(self.pos, "Function declared nogil has Python locals or temporaries")

2411
    def analyse_expressions(self, env):
2412
        self.local_scope.directives = env.directives
2413
        if self.py_func is not None:
2414
            # this will also analyse the default values
2415
            self.py_func = self.py_func.analyse_expressions(env)
2416 2417
        else:
            self.analyse_default_values(env)
2418
            self.analyse_annotations(env)
2419
        self.acquire_gil = self.need_gil_acquisition(self.local_scope)
2420
        return self
2421

Robert Bradshaw's avatar
Robert Bradshaw committed
2422 2423 2424
    def needs_assignment_synthesis(self, env, code=None):
        return False

2425
    def generate_function_header(self, code, with_pymethdef, with_opt_args = 1, with_dispatch = 1, cname = None):
2426
        scope = self.local_scope
William Stein's avatar
William Stein committed
2427 2428
        arg_decls = []
        type = self.type
2429
        for arg in type.args[:len(type.args)-type.optional_arg_count]:
2430 2431 2432 2433 2434
            arg_decl = arg.declaration_code()
            entry = scope.lookup(arg.name)
            if not entry.cf_used:
                arg_decl = 'CYTHON_UNUSED %s' % arg_decl
            arg_decls.append(arg_decl)
2435
        if with_dispatch and self.overridable:
2436 2437 2438 2439 2440 2441
            dispatch_arg = PyrexTypes.c_int_type.declaration_code(
                Naming.skip_dispatch_cname)
            if self.override:
                arg_decls.append(dispatch_arg)
            else:
                arg_decls.append('CYTHON_UNUSED %s' % dispatch_arg)
2442 2443
        if type.optional_arg_count and with_opt_args:
            arg_decls.append(type.op_arg_struct.declaration_code(Naming.optional_args_cname))
William Stein's avatar
William Stein committed
2444 2445 2446 2447
        if type.has_varargs:
            arg_decls.append("...")
        if not arg_decls:
            arg_decls = ["void"]
2448 2449
        if cname is None:
            cname = self.entry.func_cname
2450
        entity = type.function_header_code(cname, ', '.join(arg_decls))
2451
        if self.entry.visibility == 'private' and '::' not in cname:
2452
            storage_class = "static "
William Stein's avatar
William Stein committed
2453
        else:
2454
            storage_class = ""
2455
        dll_linkage = None
2456
        modifiers = code.build_function_modifiers(self.entry.func_modifiers)
Robert Bradshaw's avatar
Robert Bradshaw committed
2457

2458 2459
        header = self.return_type.declaration_code(entity, dll_linkage=dll_linkage)
        #print (storage_class, modifiers, header)
2460
        needs_proto = self.is_c_class_method
2461
        if self.template_declaration:
2462 2463
            if needs_proto:
                code.globalstate.parts['module_declarations'].putln(self.template_declaration)
2464
            code.putln(self.template_declaration)
2465 2466
        if needs_proto:
            code.globalstate.parts['module_declarations'].putln("%s%s%s; /* proto*/" % (storage_class, modifiers, header))
2467
        code.putln("%s%s%s {" % (storage_class, modifiers, header))
William Stein's avatar
William Stein committed
2468 2469

    def generate_argument_declarations(self, env, code):
2470
        scope = self.local_scope
2471
        for arg in self.args:
2472
            if arg.default:
2473
                entry = scope.lookup(arg.name)
2474
                if self.override or entry.cf_used:
2475 2476 2477
                    result = arg.calculate_default_value_code(code)
                    code.putln('%s = %s;' % (
                        arg.type.declaration_code(arg.cname), result))
2478

William Stein's avatar
William Stein committed
2479 2480
    def generate_keyword_list(self, code):
        pass
2481

2482
    def generate_argument_parsing_code(self, env, code):
2483
        i = 0
2484
        used = 0
2485
        scope = self.local_scope
2486 2487
        if self.type.optional_arg_count:
            code.putln('if (%s) {' % Naming.optional_args_cname)
2488
            for arg in self.args:
2489
                if arg.default:
2490
                    entry = scope.lookup(arg.name)
2491
                    if self.override or entry.cf_used:
2492 2493 2494 2495 2496 2497 2498 2499 2500 2501
                        code.putln('if (%s->%sn > %s) {' %
                                   (Naming.optional_args_cname,
                                    Naming.pyrex_prefix, i))
                        declarator = arg.declarator
                        while not hasattr(declarator, 'name'):
                            declarator = declarator.base
                        code.putln('%s = %s->%s;' %
                                   (arg.cname, Naming.optional_args_cname,
                                    self.type.opt_arg_cname(declarator.name)))
                        used += 1
2502
                    i += 1
2503
            for _ in range(used):
2504
                code.putln('}')
2505
            code.putln('}')
2506

2507 2508 2509 2510 2511 2512 2513 2514 2515 2516
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure and not arg.default:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(scope.lookup_here(arg.name))


William Stein's avatar
William Stein committed
2517 2518
    def generate_argument_conversion_code(self, code):
        pass
2519

William Stein's avatar
William Stein committed
2520
    def generate_argument_type_tests(self, code):
2521 2522 2523 2524 2525
        # Generate type tests for args whose type in a parent
        # class is a supertype of the declared type.
        for arg in self.type.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
2526 2527
            elif arg.type.is_pyobject and not arg.accept_none:
                self.generate_arg_none_check(arg, code)
2528

2529 2530 2531 2532 2533
    def generate_execution_code(self, code):
        super(CFuncDefNode, self).generate_execution_code(code)
        if self.py_func_stat:
            self.py_func_stat.generate_execution_code(code)

William Stein's avatar
William Stein committed
2534 2535 2536 2537
    def error_value(self):
        if self.return_type.is_pyobject:
            return "0"
        else:
2538 2539
            #return None
            return self.entry.type.exception_value
2540

William Stein's avatar
William Stein committed
2541
    def caller_will_check_exceptions(self):
2542
        return self.entry.type.exception_check
2543

2544 2545
    def generate_wrapper_functions(self, code):
        # If the C signature of a function has changed, we need to generate
2546
        # wrappers to put in the slots here.
2547 2548 2549 2550 2551 2552 2553
        k = 0
        entry = self.entry
        func_type = entry.type
        while entry.prev_entry is not None:
            k += 1
            entry = entry.prev_entry
            entry.func_cname = "%s%swrap_%s" % (self.entry.func_cname, Naming.pyrex_prefix, k)
2554
            code.putln()
2555
            self.generate_function_header(code,
2556
                                          0,
2557 2558
                                          with_dispatch = entry.type.is_overridable,
                                          with_opt_args = entry.type.optional_arg_count,
2559
                                          cname = entry.func_cname)
2560 2561 2562 2563
            if not self.return_type.is_void:
                code.put('return ')
            args = self.type.args
            arglist = [arg.cname for arg in args[:len(args)-self.type.optional_arg_count]]
2564 2565 2566 2567 2568 2569 2570 2571
            if entry.type.is_overridable:
                arglist.append(Naming.skip_dispatch_cname)
            elif func_type.is_overridable:
                arglist.append('0')
            if entry.type.optional_arg_count:
                arglist.append(Naming.optional_args_cname)
            elif func_type.optional_arg_count:
                arglist.append('NULL')
2572 2573
            code.putln('%s(%s);' % (self.entry.func_cname, ', '.join(arglist)))
            code.putln('}')
2574

William Stein's avatar
William Stein committed
2575 2576 2577 2578 2579

class PyArgDeclNode(Node):
    # Argument which must be a Python object (used
    # for * and ** arguments).
    #
2580 2581 2582
    # name        string
    # entry       Symtab.Entry
    # annotation  ExprNode or None   Py3 argument annotation
2583
    child_attrs = []
2584 2585
    is_self_arg = False
    is_type_arg = False
2586 2587 2588

    def generate_function_definitions(self, env, code):
        self.entry.generate_function_definitions(env, code)
2589 2590 2591 2592

class DecoratorNode(Node):
    # A decorator
    #
2593
    # decorator    NameNode or CallNode or AttributeNode
2594 2595
    child_attrs = ['decorator']

William Stein's avatar
William Stein committed
2596 2597 2598 2599 2600

class DefNode(FuncDefNode):
    # A Python function definition.
    #
    # name          string                 the Python name of the function
Stefan Behnel's avatar
Stefan Behnel committed
2601
    # lambda_name   string                 the internal name of a lambda 'function'
2602
    # decorators    [DecoratorNode]        list of decorators
William Stein's avatar
William Stein committed
2603
    # args          [CArgDeclNode]         formal arguments
2604
    # doc           EncodedString or None
William Stein's avatar
William Stein committed
2605
    # body          StatListNode
2606 2607
    # return_type_annotation
    #               ExprNode or None       the Py3 return type annotation
William Stein's avatar
William Stein committed
2608 2609 2610 2611
    #
    #  The following subnode is constructed internally
    #  when the def statement is inside a Python class definition.
    #
2612 2613 2614
    #  fused_py_func        DefNode     The original fused cpdef DefNode
    #                                   (in case this is a specialization)
    #  specialized_cpdefs   [DefNode]   list of specialized cpdef DefNodes
2615
    #  py_cfunc_node  PyCFunctionNode/InnerFunctionNode   The PyCFunction to create and assign
2616 2617
    #
    # decorator_indirection IndirectionNode Used to remove __Pyx_Method_ClassMethod for fused functions
2618

2619
    child_attrs = ["args", "star_arg", "starstar_arg", "body", "decorators", "return_type_annotation"]
2620

Stefan Behnel's avatar
Stefan Behnel committed
2621
    lambda_name = None
2622
    reqd_kw_flags_cname = "0"
2623
    is_wrapper = 0
2624
    no_assignment_synthesis = 0
2625
    decorators = None
2626
    return_type_annotation = None
2627
    entry = None
Robert Bradshaw's avatar
Robert Bradshaw committed
2628
    acquire_gil = 0
2629
    self_in_stararg = 0
2630
    py_cfunc_node = None
2631
    requires_classobj = False
2632
    defaults_struct = None # Dynamic kwrds structure name
2633
    doc = None
2634

2635 2636
    fused_py_func = False
    specialized_cpdefs = None
2637 2638 2639
    py_wrapper = None
    py_wrapper_required = True
    func_cname = None
2640

2641 2642
    defaults_getter = None

2643 2644
    def __init__(self, pos, **kwds):
        FuncDefNode.__init__(self, pos, **kwds)
2645
        k = rk = r = 0
2646 2647
        for arg in self.args:
            if arg.kw_only:
2648
                k += 1
2649
                if not arg.default:
2650 2651 2652 2653 2654 2655
                    rk += 1
            if not arg.default:
                r += 1
        self.num_kwonly_args = k
        self.num_required_kw_args = rk
        self.num_required_args = r
2656

2657
    def as_cfunction(self, cfunc=None, scope=None, overridable=True, returns=None, modifiers=None):
2658 2659 2660 2661
        if self.star_arg:
            error(self.star_arg.pos, "cdef function cannot have star argument")
        if self.starstar_arg:
            error(self.starstar_arg.pos, "cdef function cannot have starstar argument")
2662 2663 2664 2665 2666 2667 2668 2669 2670 2671 2672 2673 2674 2675 2676
        if cfunc is None:
            cfunc_args = []
            for formal_arg in self.args:
                name_declarator, type = formal_arg.analyse(scope, nonempty=1)
                cfunc_args.append(PyrexTypes.CFuncTypeArg(name = name_declarator.name,
                                                          cname = None,
                                                          type = py_object_type,
                                                          pos = formal_arg.pos))
            cfunc_type = PyrexTypes.CFuncType(return_type = py_object_type,
                                              args = cfunc_args,
                                              has_varargs = False,
                                              exception_value = None,
                                              exception_check = False,
                                              nogil = False,
                                              with_gil = False,
Haoyu Bai's avatar
Haoyu Bai committed
2677
                                              is_overridable = overridable)
2678
            cfunc = CVarDefNode(self.pos, type=cfunc_type)
2679
        else:
2680 2681
            if scope is None:
                scope = cfunc.scope
2682 2683 2684
            cfunc_type = cfunc.type
            if len(self.args) != len(cfunc_type.args) or cfunc_type.has_varargs:
                error(self.pos, "wrong number of arguments")
Stefan Behnel's avatar
Stefan Behnel committed
2685
                error(cfunc.pos, "previous declaration here")
2686 2687 2688
            for i, (formal_arg, type_arg) in enumerate(zip(self.args, cfunc_type.args)):
                name_declarator, type = formal_arg.analyse(scope, nonempty=1,
                                                           is_self_arg = (i == 0 and scope.is_c_class_scope))
Stefan Behnel's avatar
Stefan Behnel committed
2689
                if type is None or type is PyrexTypes.py_object_type:
2690 2691
                    formal_arg.type = type_arg.type
                    formal_arg.name_declarator = name_declarator
2692
        from . import ExprNodes
2693
        if cfunc_type.exception_value is None:
2694 2695
            exception_value = None
        else:
2696
            exception_value = ExprNodes.ConstNode(self.pos, value=cfunc_type.exception_value, type=cfunc_type.return_type)
2697
        declarator = CFuncDeclaratorNode(self.pos,
2698 2699 2700
                                         base = CNameDeclaratorNode(self.pos, name=self.name, cname=None),
                                         args = self.args,
                                         has_varargs = False,
2701
                                         exception_check = cfunc_type.exception_check,
2702
                                         exception_value = exception_value,
2703 2704
                                         with_gil = cfunc_type.with_gil,
                                         nogil = cfunc_type.nogil)
2705
        return CFuncDefNode(self.pos,
2706
                            modifiers = modifiers or [],
2707
                            base_type = CAnalysedBaseTypeNode(self.pos, type=cfunc_type.return_type),
2708 2709 2710
                            declarator = declarator,
                            body = self.body,
                            doc = self.doc,
2711 2712 2713 2714
                            overridable = cfunc_type.is_overridable,
                            type = cfunc_type,
                            with_gil = cfunc_type.with_gil,
                            nogil = cfunc_type.nogil,
2715
                            visibility = 'private',
2716
                            api = False,
2717 2718
                            directive_locals = getattr(cfunc, 'directive_locals', {}),
                            directive_returns = returns)
2719

2720 2721 2722 2723 2724 2725 2726 2727 2728 2729 2730
    def is_cdef_func_compatible(self):
        """Determines if the function's signature is compatible with a
        cdef function.  This can be used before calling
        .as_cfunction() to see if that will be successful.
        """
        if self.needs_closure:
            return False
        if self.star_arg or self.starstar_arg:
            return False
        return True

William Stein's avatar
William Stein committed
2731
    def analyse_declarations(self, env):
2732 2733 2734 2735 2736 2737 2738 2739 2740 2741 2742 2743 2744 2745 2746
        self.is_classmethod = self.is_staticmethod = False
        if self.decorators:
            for decorator in self.decorators:
                func = decorator.decorator
                if func.is_name:
                    self.is_classmethod |= func.name == 'classmethod'
                    self.is_staticmethod |= func.name == 'staticmethod'

        if self.is_classmethod and env.lookup_here('classmethod'):
            # classmethod() was overridden - not much we can do here ...
            self.is_classmethod = False
        if self.is_staticmethod and env.lookup_here('staticmethod'):
            # staticmethod() was overridden - not much we can do here ...
            self.is_staticmethod = False

2747
        if self.name == '__new__' and env.is_py_class_scope:
Vitja Makarov's avatar
Vitja Makarov committed
2748 2749
            self.is_staticmethod = 1

2750
        self.analyse_argument_types(env)
2751 2752 2753 2754
        if self.name == '<lambda>':
            self.declare_lambda_function(env)
        else:
            self.declare_pyfunction(env)
2755

2756 2757
        self.analyse_signature(env)
        self.return_type = self.entry.signature.return_type()
2758 2759 2760 2761 2762 2763 2764
        # if a signature annotation provides a more specific return object type, use it
        if self.return_type is py_object_type and self.return_type_annotation:
            if env.directives['annotation_typing'] and not self.entry.is_special:
                _, return_type = _analyse_signature_annotation(self.return_type_annotation, env)
                if return_type and return_type.is_pyobject:
                    self.return_type = return_type

2765 2766
        self.create_local_scope(env)

2767 2768 2769 2770 2771 2772 2773 2774 2775 2776
        self.py_wrapper = DefNodeWrapper(
            self.pos,
            target=self,
            name=self.entry.name,
            args=self.args,
            star_arg=self.star_arg,
            starstar_arg=self.starstar_arg,
            return_type=self.return_type)
        self.py_wrapper.analyse_declarations(env)

2777
    def analyse_argument_types(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
2778
        self.directive_locals = env.directives['locals']
2779
        allow_none_for_extension_args = env.directives['allow_none_for_extension_args']
2780 2781 2782 2783

        f2s = env.fused_to_specific
        env.fused_to_specific = None

William Stein's avatar
William Stein committed
2784
        for arg in self.args:
2785 2786 2787 2788 2789 2790 2791
            if hasattr(arg, 'name'):
                name_declarator = None
            else:
                base_type = arg.base_type.analyse(env)
                name_declarator, type = \
                    arg.declarator.analyse(base_type, env)
                arg.name = name_declarator.name
2792
                arg.type = type
2793 2794 2795 2796

                if type.is_fused:
                    self.has_fused_arguments = True

2797
            self.align_argument_type(env, arg)
2798
            if name_declarator and name_declarator.cname:
William Stein's avatar
William Stein committed
2799 2800
                error(self.pos,
                    "Python function argument cannot have C name specification")
2801
            arg.type = arg.type.as_argument_type()
William Stein's avatar
William Stein committed
2802 2803 2804 2805
            arg.hdr_type = None
            arg.needs_conversion = 0
            arg.needs_type_test = 0
            arg.is_generic = 1
2806
            if arg.type.is_pyobject or arg.type.is_buffer or arg.type.is_memoryviewslice:
2807 2808 2809 2810
                if arg.or_none:
                    arg.accept_none = True
                elif arg.not_none:
                    arg.accept_none = False
2811
                elif (arg.type.is_extension_type or arg.type.is_builtin_type
2812
                      or arg.type.is_buffer or arg.type.is_memoryviewslice):
2813 2814 2815 2816 2817 2818
                    if arg.default and arg.default.constant_result is None:
                        # special case: def func(MyType obj = None)
                        arg.accept_none = True
                    else:
                        # default depends on compiler directive
                        arg.accept_none = allow_none_for_extension_args
2819 2820 2821
                else:
                    # probably just a plain 'object'
                    arg.accept_none = True
2822
            else:
2823
                arg.accept_none = True # won't be used, but must be there
2824
                if arg.not_none:
2825
                    error(arg.pos, "Only Python type arguments can have 'not None'")
2826
                if arg.or_none:
2827
                    error(arg.pos, "Only Python type arguments can have 'or None'")
2828 2829
        env.fused_to_specific = f2s

William Stein's avatar
William Stein committed
2830
    def analyse_signature(self, env):
2831
        if self.entry.is_special:
2832
            if self.decorators:
2833
                error(self.pos, "special functions of cdef classes cannot have decorators")
2834 2835 2836 2837
            self.entry.trivial_signature = len(self.args) == 1 and not (self.star_arg or self.starstar_arg)
        elif not env.directives['always_allow_keywords'] and not (self.star_arg or self.starstar_arg):
            # Use the simpler calling signature for zero- and one-argument functions.
            if self.entry.signature is TypeSlots.pyfunction_signature:
2838 2839
                if len(self.args) == 0:
                    self.entry.signature = TypeSlots.pyfunction_noargs
2840 2841 2842
                elif len(self.args) == 1:
                    if self.args[0].default is None and not self.args[0].kw_only:
                        self.entry.signature = TypeSlots.pyfunction_onearg
2843 2844 2845
            elif self.entry.signature is TypeSlots.pymethod_signature:
                if len(self.args) == 1:
                    self.entry.signature = TypeSlots.unaryfunc
2846 2847 2848
                elif len(self.args) == 2:
                    if self.args[1].default is None and not self.args[1].kw_only:
                        self.entry.signature = TypeSlots.ibinaryfunc
2849

William Stein's avatar
William Stein committed
2850 2851
        sig = self.entry.signature
        nfixed = sig.num_fixed_args()
2852 2853 2854 2855 2856 2857 2858 2859 2860
        if sig is TypeSlots.pymethod_signature and nfixed == 1 \
               and len(self.args) == 0 and self.star_arg:
            # this is the only case where a diverging number of
            # arguments is not an error - when we have no explicit
            # 'self' parameter as in method(*args)
            sig = self.entry.signature = TypeSlots.pyfunction_signature # self is not 'really' used
            self.self_in_stararg = 1
            nfixed = 0

2861 2862
        if self.is_staticmethod and env.is_c_class_scope:
            nfixed = 0
2863
            self.self_in_stararg = True  # FIXME: why for staticmethods?
2864 2865 2866 2867 2868 2869

            self.entry.signature = sig = copy.copy(sig)
            sig.fixed_arg_format = "*"
            sig.is_staticmethod = True
            sig.has_generic_args = True

2870 2871
        if ((self.is_classmethod or self.is_staticmethod) and
            self.has_fused_arguments and env.is_c_class_scope):
2872 2873
            del self.decorator_indirection.stats[:]

2874 2875 2876 2877 2878 2879 2880
        for i in range(min(nfixed, len(self.args))):
            arg = self.args[i]
            arg.is_generic = 0
            if sig.is_self_arg(i) and not self.is_staticmethod:
                if self.is_classmethod:
                    arg.is_type_arg = 1
                    arg.hdr_type = arg.type = Builtin.type_type
William Stein's avatar
William Stein committed
2881
                else:
2882 2883 2884
                    arg.is_self_arg = 1
                    arg.hdr_type = arg.type = env.parent_type
                arg.needs_conversion = 0
William Stein's avatar
William Stein committed
2885
            else:
2886 2887 2888 2889 2890 2891 2892 2893 2894 2895 2896 2897 2898 2899 2900
                arg.hdr_type = sig.fixed_arg_type(i)
                if not arg.type.same_as(arg.hdr_type):
                    if arg.hdr_type.is_pyobject and arg.type.is_pyobject:
                        arg.needs_type_test = 1
                    else:
                        arg.needs_conversion = 1
            if arg.needs_conversion:
                arg.hdr_cname = Naming.arg_prefix + arg.name
            else:
                arg.hdr_cname = Naming.var_prefix + arg.name

        if nfixed > len(self.args):
            self.bad_signature()
            return
        elif nfixed < len(self.args):
William Stein's avatar
William Stein committed
2901 2902 2903
            if not sig.has_generic_args:
                self.bad_signature()
            for arg in self.args:
Robert Bradshaw's avatar
Robert Bradshaw committed
2904 2905
                if arg.is_generic and \
                        (arg.type.is_extension_type or arg.type.is_builtin_type):
William Stein's avatar
William Stein committed
2906
                    arg.needs_type_test = 1
2907

William Stein's avatar
William Stein committed
2908 2909 2910 2911
    def bad_signature(self):
        sig = self.entry.signature
        expected_str = "%d" % sig.num_fixed_args()
        if sig.has_generic_args:
Stefan Behnel's avatar
Stefan Behnel committed
2912
            expected_str += " or more"
William Stein's avatar
William Stein committed
2913 2914 2915 2916 2917 2918 2919 2920 2921
        name = self.name
        if name.startswith("__") and name.endswith("__"):
            desc = "Special method"
        else:
            desc = "Method"
        error(self.pos,
            "%s %s has wrong number of arguments "
            "(%d declared, %s expected)" % (
                desc, self.name, len(self.args), expected_str))
2922

William Stein's avatar
William Stein committed
2923
    def declare_pyfunction(self, env):
2924 2925
        #print "DefNode.declare_pyfunction:", self.name, "in", env ###
        name = self.name
Stefan Behnel's avatar
Stefan Behnel committed
2926
        entry = env.lookup_here(name)
2927 2928
        if entry:
            if entry.is_final_cmethod and not env.parent_type.is_final_type:
Stefan Behnel's avatar
Stefan Behnel committed
2929
                error(self.pos, "Only final types can have final Python (def/cpdef) methods")
2930 2931 2932
            if (entry.type.is_cfunction and not entry.is_builtin_cmethod
                and not self.is_wrapper):
                warning(self.pos, "Overriding cdef method with def method.", 5)
2933
        entry = env.declare_pyfunction(name, self.pos, allow_redefine=not self.is_wrapper)
2934
        self.entry = entry
2935
        prefix = env.next_id(env.scope_prefix)
2936
        self.entry.pyfunc_cname = Naming.pyfunc_prefix + prefix + name
2937 2938
        if Options.docstrings:
            entry.doc = embed_position(self.pos, self.doc)
2939
            entry.doc_cname = Naming.funcdoc_prefix + prefix + name
2940
            if entry.is_special:
2941
                if entry.name in TypeSlots.invisible or not entry.doc or (entry.name in '__getattr__' and env.directives['fast_getattr']):
2942 2943 2944
                    entry.wrapperbase_cname = None
                else:
                    entry.wrapperbase_cname = Naming.wrapperbase_prefix + prefix + name
2945 2946
        else:
            entry.doc = None
2947

Stefan Behnel's avatar
Stefan Behnel committed
2948
    def declare_lambda_function(self, env):
2949
        entry = env.declare_lambda_function(self.lambda_name, self.pos)
Stefan Behnel's avatar
Stefan Behnel committed
2950 2951
        entry.doc = None
        self.entry = entry
2952
        self.entry.pyfunc_cname = entry.cname
Stefan Behnel's avatar
Stefan Behnel committed
2953

William Stein's avatar
William Stein committed
2954 2955 2956 2957 2958 2959 2960 2961 2962 2963
    def declare_arguments(self, env):
        for arg in self.args:
            if not arg.name:
                error(arg.pos, "Missing argument name")
            if arg.needs_conversion:
                arg.entry = env.declare_var(arg.name, arg.type, arg.pos)
                if arg.type.is_pyobject:
                    arg.entry.init = "0"
            else:
                arg.entry = self.declare_argument(env, arg)
2964
            arg.entry.is_arg = 1
2965
            arg.entry.used = 1
William Stein's avatar
William Stein committed
2966 2967 2968 2969 2970 2971
            arg.entry.is_self_arg = arg.is_self_arg
        self.declare_python_arg(env, self.star_arg)
        self.declare_python_arg(env, self.starstar_arg)

    def declare_python_arg(self, env, arg):
        if arg:
2972
            if env.directives['infer_types'] != False:
2973 2974 2975 2976
                type = PyrexTypes.unspecified_type
            else:
                type = py_object_type
            entry = env.declare_var(arg.name, type, arg.pos)
2977
            entry.is_arg = 1
2978 2979 2980 2981
            entry.used = 1
            entry.init = "0"
            entry.xdecref_cleanup = 1
            arg.entry = entry
2982

William Stein's avatar
William Stein committed
2983
    def analyse_expressions(self, env):
2984
        self.local_scope.directives = env.directives
William Stein's avatar
William Stein committed
2985
        self.analyse_default_values(env)
2986 2987 2988
        self.analyse_annotations(env)
        if self.return_type_annotation:
            self.return_type_annotation = self.return_type_annotation.analyse_types(env)
2989

2990
        if not self.needs_assignment_synthesis(env) and self.decorators:
2991
            for decorator in self.decorators[::-1]:
2992
                decorator.decorator = decorator.decorator.analyse_expressions(env)
2993

2994
        self.py_wrapper.prepare_argument_coercion(env)
2995
        return self
2996

2997
    def needs_assignment_synthesis(self, env, code=None):
2998
        if self.is_staticmethod:
2999
            return True
3000 3001
        if self.is_wrapper or self.specialized_cpdefs or self.entry.is_fused_specialized:
            return False
3002 3003
        if self.no_assignment_synthesis:
            return False
3004
        # Should enable for module level as well, that will require more testing...
3005
        if self.entry.is_anonymous:
3006
            return True
3007 3008 3009 3010 3011 3012 3013
        if env.is_module_scope:
            if code is None:
                return env.directives['binding']
            else:
                return code.globalstate.directives['binding']
        return env.is_py_class_scope or env.is_closure_scope

3014 3015 3016 3017
    def error_value(self):
        return self.entry.signature.error_value

    def caller_will_check_exceptions(self):
3018
        return self.entry.signature.exception_check
3019 3020

    def generate_function_definitions(self, env, code):
3021 3022 3023
        if self.defaults_getter:
            self.defaults_getter.generate_function_definitions(env, code)

3024 3025 3026 3027 3028 3029 3030
        # Before closure cnames are mangled
        if self.py_wrapper_required:
            # func_cname might be modified by @cname
            self.py_wrapper.func_cname = self.entry.func_cname
            self.py_wrapper.generate_function_definitions(env, code)
        FuncDefNode.generate_function_definitions(self, env, code)

3031
    def generate_function_header(self, code, with_pymethdef, proto_only=0):
3032 3033 3034 3035 3036
        if proto_only:
            if self.py_wrapper_required:
                self.py_wrapper.generate_function_header(
                    code, with_pymethdef, True)
            return
William Stein's avatar
William Stein committed
3037
        arg_code_list = []
3038
        if self.entry.signature.has_dummy_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3039 3040 3041
            self_arg = 'PyObject *%s' % Naming.self_cname
            if not self.needs_outer_scope:
                self_arg = 'CYTHON_UNUSED ' + self_arg
3042 3043 3044 3045 3046 3047 3048 3049
            arg_code_list.append(self_arg)

        def arg_decl_code(arg):
            entry = arg.entry
            if entry.in_closure:
                cname = entry.original_cname
            else:
                cname = entry.cname
3050
            decl = entry.type.declaration_code(cname)
Stefan Behnel's avatar
Stefan Behnel committed
3051 3052 3053
            if not entry.cf_used:
                decl = 'CYTHON_UNUSED ' + decl
            return decl
3054 3055 3056 3057 3058 3059 3060 3061 3062 3063 3064 3065 3066 3067 3068 3069 3070 3071 3072 3073 3074 3075 3076 3077 3078 3079 3080 3081 3082 3083 3084 3085 3086 3087 3088 3089 3090 3091 3092 3093 3094 3095 3096 3097 3098 3099 3100 3101 3102 3103 3104 3105 3106 3107 3108 3109 3110 3111 3112 3113 3114 3115 3116 3117 3118 3119

        for arg in self.args:
            arg_code_list.append(arg_decl_code(arg))
        if self.star_arg:
            arg_code_list.append(arg_decl_code(self.star_arg))
        if self.starstar_arg:
            arg_code_list.append(arg_decl_code(self.starstar_arg))
        arg_code = ', '.join(arg_code_list)
        dc = self.return_type.declaration_code(self.entry.pyfunc_cname)

        decls_code = code.globalstate['decls']
        preprocessor_guard = self.get_preprocessor_guard()
        if preprocessor_guard:
            decls_code.putln(preprocessor_guard)
        decls_code.putln(
            "static %s(%s); /* proto */" % (dc, arg_code))
        if preprocessor_guard:
            decls_code.putln("#endif")
        code.putln("static %s(%s) {" % (dc, arg_code))

    def generate_argument_declarations(self, env, code):
        pass

    def generate_keyword_list(self, code):
        pass

    def generate_argument_parsing_code(self, env, code):
        # Move arguments into closure if required
        def put_into_closure(entry):
            if entry.in_closure:
                code.putln('%s = %s;' % (entry.cname, entry.original_cname))
                code.put_var_incref(entry)
                code.put_var_giveref(entry)
        for arg in self.args:
            put_into_closure(arg.entry)
        for arg in self.star_arg, self.starstar_arg:
            if arg:
                put_into_closure(arg.entry)

    def generate_argument_type_tests(self, code):
        pass


class DefNodeWrapper(FuncDefNode):
    # DefNode python wrapper code generator

    defnode = None
    target = None # Target DefNode

    def __init__(self, *args, **kwargs):
        FuncDefNode.__init__(self, *args, **kwargs)
        self.num_kwonly_args = self.target.num_kwonly_args
        self.num_required_kw_args = self.target.num_required_kw_args
        self.num_required_args = self.target.num_required_args
        self.self_in_stararg = self.target.self_in_stararg
        self.signature = None

    def analyse_declarations(self, env):
        target_entry = self.target.entry
        name = self.name
        prefix = env.next_id(env.scope_prefix)
        target_entry.func_cname = Naming.pywrap_prefix + prefix + name
        target_entry.pymethdef_cname = Naming.pymethdef_prefix + prefix + name

        self.signature = target_entry.signature

3120
    def prepare_argument_coercion(self, env):
3121 3122 3123 3124 3125 3126 3127 3128
        # This is only really required for Cython utility code at this time,
        # everything else can be done during code generation.  But we expand
        # all utility code here, simply because we cannot easily distinguish
        # different code types.
        for arg in self.args:
            if not arg.type.is_pyobject:
                if not arg.type.create_from_py_utility_code(env):
                    pass # will fail later
Robert Bradshaw's avatar
Robert Bradshaw committed
3129
            elif arg.hdr_type and not arg.hdr_type.is_pyobject:
3130 3131
                if not arg.hdr_type.create_to_py_utility_code(env):
                    pass # will fail later
3132

3133 3134 3135 3136 3137 3138 3139 3140 3141 3142 3143 3144 3145 3146 3147 3148
    def signature_has_nongeneric_args(self):
        argcount = len(self.args)
        if argcount == 0 or (
                argcount == 1 and (self.args[0].is_self_arg or
                                   self.args[0].is_type_arg)):
            return 0
        return 1

    def signature_has_generic_args(self):
        return self.signature.has_generic_args

    def generate_function_body(self, code):
        args = []
        if self.signature.has_dummy_arg:
            args.append(Naming.self_cname)
        for arg in self.args:
3149 3150 3151
            if arg.hdr_type and not (arg.type.is_memoryviewslice or
                                     arg.type.is_struct or
                                     arg.type.is_complex):
3152 3153 3154
                args.append(arg.type.cast_code(arg.entry.cname))
            else:
                args.append(arg.entry.cname)
3155 3156 3157 3158 3159 3160 3161 3162 3163 3164 3165 3166 3167
        if self.star_arg:
            args.append(self.star_arg.entry.cname)
        if self.starstar_arg:
            args.append(self.starstar_arg.entry.cname)
        args = ', '.join(args)
        if not self.return_type.is_void:
            code.put('%s = ' % Naming.retval_cname)
        code.putln('%s(%s);' % (
            self.target.entry.pyfunc_cname, args))

    def generate_function_definitions(self, env, code):
        lenv = self.target.local_scope
        # Generate C code for header and body of function
3168
        code.mark_pos(self.pos)
3169 3170 3171 3172 3173 3174 3175 3176 3177
        code.putln("")
        code.putln("/* Python wrapper */")
        preprocessor_guard = self.target.get_preprocessor_guard()
        if preprocessor_guard:
            code.putln(preprocessor_guard)

        code.enter_cfunc_scope()
        code.return_from_error_cleanup_label = code.new_label()

Vitja Makarov's avatar
Vitja Makarov committed
3178
        with_pymethdef = (self.target.needs_assignment_synthesis(env, code) or
3179 3180 3181 3182 3183 3184 3185 3186 3187 3188 3189 3190 3191 3192 3193 3194 3195 3196 3197 3198 3199 3200 3201
                          self.target.pymethdef_required)
        self.generate_function_header(code, with_pymethdef)
        self.generate_argument_declarations(lenv, code)
        tempvardecl_code = code.insertion_point()

        if self.return_type.is_pyobject:
            retval_init = ' = 0'
        else:
            retval_init = ''
        if not self.return_type.is_void:
            code.putln('%s%s;' % (
                self.return_type.declaration_code(Naming.retval_cname),
                retval_init))
        code.put_declare_refcount_context()
        code.put_setup_refcount_context('%s (wrapper)' % self.name)

        self.generate_argument_parsing_code(lenv, code)
        self.generate_argument_type_tests(code)
        self.generate_function_body(code)

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)

3202 3203 3204 3205
        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

3206 3207 3208 3209 3210 3211
        # ----- Error cleanup
        if code.error_label in code.labels_used:
            code.put_goto(code.return_label)
            code.put_label(code.error_label)
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
3212 3213 3214
            err_val = self.error_value()
            if err_val is not None:
                code.putln("%s = %s;" % (Naming.retval_cname, err_val))
3215 3216 3217 3218 3219 3220 3221 3222 3223 3224 3225 3226 3227 3228 3229 3230 3231 3232 3233

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
        for entry in lenv.var_entries:
            if entry.is_arg and entry.type.is_pyobject:
                code.put_var_decref(entry)

        code.put_finish_refcount_context()
        if not self.return_type.is_void:
            code.putln("return %s;" % Naming.retval_cname)
        code.putln('}')
        code.exit_cfunc_scope()
        if preprocessor_guard:
            code.putln("#endif /*!(%s)*/" % preprocessor_guard)

    def generate_function_header(self, code, with_pymethdef, proto_only=0):
        arg_code_list = []
        sig = self.signature

3234
        if sig.has_dummy_arg or self.self_in_stararg:
3235 3236 3237 3238
            arg_code = "PyObject *%s" % Naming.self_cname
            if not sig.has_dummy_arg:
                arg_code = 'CYTHON_UNUSED ' + arg_code
            arg_code_list.append(arg_code)
3239

William Stein's avatar
William Stein committed
3240 3241
        for arg in self.args:
            if not arg.is_generic:
3242
                if arg.is_self_arg or arg.is_type_arg:
William Stein's avatar
William Stein committed
3243 3244
                    arg_code_list.append("PyObject *%s" % arg.hdr_cname)
                else:
3245 3246 3247 3248
                    arg_code_list.append(
                        arg.hdr_type.declaration_code(arg.hdr_cname))
        entry = self.target.entry
        if not entry.is_special and sig.method_flags() == [TypeSlots.method_noargs]:
3249
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
3250
        if entry.scope.is_c_class_scope and entry.name == "__ipow__":
Lisandro Dalcin's avatar
Lisandro Dalcin committed
3251
            arg_code_list.append("CYTHON_UNUSED PyObject *unused")
William Stein's avatar
William Stein committed
3252 3253 3254 3255 3256
        if sig.has_generic_args:
            arg_code_list.append(
                "PyObject *%s, PyObject *%s"
                    % (Naming.args_cname, Naming.kwds_cname))
        arg_code = ", ".join(arg_code_list)
3257 3258 3259 3260 3261 3262

        # Prevent warning: unused function '__pyx_pw_5numpy_7ndarray_1__getbuffer__'
        mf = ""
        if (entry.name in ("__getbuffer__", "__releasebuffer__")
            and entry.scope.is_c_class_scope):
            mf = "CYTHON_UNUSED "
3263
            with_pymethdef = False
3264

3265
        dc = self.return_type.declaration_code(entry.func_cname)
3266
        header = "static %s%s(%s)" % (mf, dc, arg_code)
William Stein's avatar
William Stein committed
3267
        code.putln("%s; /*proto*/" % header)
3268

3269
        if proto_only:
3270
            if self.target.fused_py_func:
3271 3272 3273
                # If we are the specialized version of the cpdef, we still
                # want the prototype for the "fused cpdef", in case we're
                # checking to see if our method was overridden in Python
3274
                self.target.fused_py_func.generate_function_header(
3275
                                    code, with_pymethdef, proto_only=True)
3276
            return
3277

3278 3279 3280 3281
        if (Options.docstrings and entry.doc and
                not self.target.fused_py_func and
                not entry.scope.is_property_scope and
                (not entry.is_special or entry.wrapperbase_cname)):
3282
            # h_code = code.globalstate['h_code']
3283
            docstr = entry.doc
3284

Stefan Behnel's avatar
Stefan Behnel committed
3285
            if docstr.is_unicode:
3286
                docstr = docstr.utf8encode()
3287

William Stein's avatar
William Stein committed
3288 3289
            code.putln(
                'static char %s[] = "%s";' % (
3290
                    entry.doc_cname,
3291
                    split_string_literal(escape_byte_string(docstr))))
3292

3293
            if entry.is_special:
3294
                code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
3295
                code.putln(
3296
                    "struct wrapperbase %s;" % entry.wrapperbase_cname)
3297
                code.putln('#endif')
3298

3299
        if with_pymethdef or self.target.fused_py_func:
William Stein's avatar
William Stein committed
3300
            code.put(
3301
                "static PyMethodDef %s = " %
3302 3303
                    entry.pymethdef_cname)
            code.put_pymethoddef(self.target.entry, ";", allow_skip=False)
William Stein's avatar
William Stein committed
3304 3305 3306 3307
        code.putln("%s {" % header)

    def generate_argument_declarations(self, env, code):
        for arg in self.args:
3308
            if arg.is_generic:
3309 3310
                if arg.needs_conversion:
                    code.putln("PyObject *%s = 0;" % arg.hdr_cname)
3311
                else:
3312
                    code.put_var_declaration(arg.entry)
3313 3314 3315
        for entry in env.var_entries:
            if entry.is_arg:
                code.put_var_declaration(entry)
3316

3317
    def generate_argument_parsing_code(self, env, code):
Stefan Behnel's avatar
Stefan Behnel committed
3318 3319
        # Generate fast equivalent of PyArg_ParseTuple call for
        # generic arguments, if any, including args/kwargs
3320 3321
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
Stefan Behnel's avatar
Stefan Behnel committed
3322
        end_label = code.new_label("argument_unpacking_done")
3323

3324 3325 3326
        has_kwonly_args = self.num_kwonly_args > 0
        has_star_or_kw_args = self.star_arg is not None \
            or self.starstar_arg is not None or has_kwonly_args
3327

3328
        for arg in self.args:
3329
            if not arg.type.is_pyobject:
3330 3331
                if not arg.type.create_from_py_utility_code(env):
                    pass # will fail later
3332

3333
        if not self.signature_has_generic_args():
3334 3335
            if has_star_or_kw_args:
                error(self.pos, "This method cannot have * or keyword arguments")
3336
            self.generate_argument_conversion_code(code)
3337

3338 3339
        elif not self.signature_has_nongeneric_args():
            # func(*args) or func(**kw) or func(*args, **kw)
3340
            self.generate_stararg_copy_code(code)
3341

3342
        else:
3343
            self.generate_tuple_and_keyword_parsing_code(self.args, end_label, code)
3344

3345 3346
        code.error_label = old_error_label
        if code.label_used(our_error_label):
3347 3348
            if not code.label_used(end_label):
                code.put_goto(end_label)
3349 3350 3351 3352 3353
            code.put_label(our_error_label)
            if has_star_or_kw_args:
                self.generate_arg_decref(self.star_arg, code)
                if self.starstar_arg:
                    if self.starstar_arg.entry.xdecref_cleanup:
3354
                        code.put_var_xdecref_clear(self.starstar_arg.entry)
3355
                    else:
3356
                        code.put_var_decref_clear(self.starstar_arg.entry)
3357
            code.put_add_traceback(self.target.entry.qualified_name)
3358
            code.put_finish_refcount_context()
3359
            code.putln("return %s;" % self.error_value())
3360
        if code.label_used(end_label):
3361 3362
            code.put_label(end_label)

William Stein's avatar
William Stein committed
3363 3364
    def generate_arg_xdecref(self, arg, code):
        if arg:
3365
            code.put_var_xdecref_clear(arg.entry)
3366

3367 3368
    def generate_arg_decref(self, arg, code):
        if arg:
3369
            code.put_var_decref_clear(arg.entry)
William Stein's avatar
William Stein committed
3370

3371 3372
    def generate_stararg_copy_code(self, code):
        if not self.star_arg:
3373 3374
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3375 3376 3377
            code.putln("if (unlikely(PyTuple_GET_SIZE(%s) > 0)) {" %
                       Naming.args_cname)
            code.put('__Pyx_RaiseArgtupleInvalid("%s", 1, 0, 0, PyTuple_GET_SIZE(%s)); return %s;' % (
Stefan Behnel's avatar
Stefan Behnel committed
3378
                    self.name, Naming.args_cname, self.error_value()))
3379
            code.putln("}")
3380

3381 3382 3383 3384 3385 3386 3387 3388
        if self.starstar_arg:
            if self.star_arg:
                kwarg_check = "unlikely(%s)" % Naming.kwds_cname
            else:
                kwarg_check = "%s" % Naming.kwds_cname
        else:
            kwarg_check = "unlikely(%s) && unlikely(PyDict_Size(%s) > 0)" % (
                Naming.kwds_cname, Naming.kwds_cname)
3389 3390
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("KeywordStringCheck", "FunctionArguments.c"))
3391
        code.putln(
3392 3393
            "if (%s && unlikely(!__Pyx_CheckKeywordStrings(%s, \"%s\", %d))) return %s;" % (
                kwarg_check, Naming.kwds_cname, self.name,
3394
                bool(self.starstar_arg), self.error_value()))
3395

3396
        if self.starstar_arg:
3397
            code.putln("%s = (%s) ? PyDict_Copy(%s) : PyDict_New();" % (
3398 3399 3400
                    self.starstar_arg.entry.cname,
                    Naming.kwds_cname,
                    Naming.kwds_cname))
3401 3402
            code.putln("if (unlikely(!%s)) return %s;" % (
                    self.starstar_arg.entry.cname, self.error_value()))
3403
            self.starstar_arg.entry.xdecref_cleanup = 0
3404
            code.put_gotref(self.starstar_arg.entry.cname)
3405

3406
        if self.self_in_stararg and not self.target.is_staticmethod:
3407 3408 3409 3410 3411 3412 3413
            # need to create a new tuple with 'self' inserted as first item
            code.put("%s = PyTuple_New(PyTuple_GET_SIZE(%s)+1); if (unlikely(!%s)) " % (
                    self.star_arg.entry.cname,
                    Naming.args_cname,
                    self.star_arg.entry.cname))
            if self.starstar_arg:
                code.putln("{")
3414
                code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
3415 3416 3417 3418 3419 3420 3421 3422 3423 3424 3425 3426 3427 3428 3429 3430 3431 3432 3433 3434 3435 3436
                code.putln("return %s;" % self.error_value())
                code.putln("}")
            else:
                code.putln("return %s;" % self.error_value())
            code.put_gotref(self.star_arg.entry.cname)
            code.put_incref(Naming.self_cname, py_object_type)
            code.put_giveref(Naming.self_cname)
            code.putln("PyTuple_SET_ITEM(%s, 0, %s);" % (
                self.star_arg.entry.cname, Naming.self_cname))
            temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
            code.putln("for (%s=0; %s < PyTuple_GET_SIZE(%s); %s++) {" % (
                temp, temp, Naming.args_cname, temp))
            code.putln("PyObject* item = PyTuple_GET_ITEM(%s, %s);" % (
                Naming.args_cname, temp))
            code.put_incref("item", py_object_type)
            code.put_giveref("item")
            code.putln("PyTuple_SET_ITEM(%s, %s+1, item);" % (
                self.star_arg.entry.cname, temp))
            code.putln("}")
            code.funcstate.release_temp(temp)
            self.star_arg.entry.xdecref_cleanup = 0
        elif self.star_arg:
3437 3438 3439 3440 3441 3442
            code.put_incref(Naming.args_cname, py_object_type)
            code.putln("%s = %s;" % (
                    self.star_arg.entry.cname,
                    Naming.args_cname))
            self.star_arg.entry.xdecref_cleanup = 0

3443
    def generate_tuple_and_keyword_parsing_code(self, args, success_label, code):
3444
        argtuple_error_label = code.new_label("argtuple_error")
3445

3446 3447 3448 3449 3450 3451 3452 3453 3454 3455 3456 3457 3458 3459 3460 3461 3462 3463 3464 3465
        positional_args = []
        required_kw_only_args = []
        optional_kw_only_args = []
        for arg in args:
            if arg.is_generic:
                if arg.default:
                    if not arg.is_self_arg and not arg.is_type_arg:
                        if arg.kw_only:
                            optional_kw_only_args.append(arg)
                        else:
                            positional_args.append(arg)
                elif arg.kw_only:
                    required_kw_only_args.append(arg)
                elif not arg.is_self_arg and not arg.is_type_arg:
                    positional_args.append(arg)

        # sort required kw-only args before optional ones to avoid special
        # cases in the unpacking code
        kw_only_args = required_kw_only_args + optional_kw_only_args

3466
        min_positional_args = self.num_required_args - self.num_required_kw_args
3467
        if len(args) > 0 and (args[0].is_self_arg or args[0].is_type_arg):
3468 3469
            min_positional_args -= 1
        max_positional_args = len(positional_args)
3470 3471
        has_fixed_positional_count = not self.star_arg and \
            min_positional_args == max_positional_args
3472
        has_kw_only_args = bool(kw_only_args)
3473

Stefan Behnel's avatar
Stefan Behnel committed
3474
        if self.num_required_kw_args:
3475 3476
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseKeywordRequired", "FunctionArguments.c"))
Stefan Behnel's avatar
Stefan Behnel committed
3477

3478 3479 3480
        if self.starstar_arg or self.star_arg:
            self.generate_stararg_init_code(max_positional_args, code)

3481 3482
        code.putln('{')
        all_args = tuple(positional_args) + tuple(kw_only_args)
3483 3484 3485 3486
        code.putln("static PyObject **%s[] = {%s,0};" % (
            Naming.pykwdlist_cname,
            ','.join([ '&%s' % code.intern_identifier(arg.name)
                        for arg in all_args ])))
3487 3488 3489 3490 3491 3492 3493 3494 3495 3496

        # Before being converted and assigned to the target variables,
        # borrowed references to all unpacked argument values are
        # collected into a local PyObject* array called "values",
        # regardless if they were taken from default arguments,
        # positional arguments or keyword arguments.  Note that
        # C-typed default arguments are handled at conversion time,
        # so their array value is NULL in the end if no argument
        # was passed for them.
        self.generate_argument_values_setup_code(all_args, code)
3497

3498
        # --- optimised code when we receive keyword arguments
3499 3500 3501
        code.putln("if (%s(%s)) {" % (
            (self.num_required_kw_args > 0) and "likely" or "unlikely",
            Naming.kwds_cname))
3502 3503
        self.generate_keyword_unpacking_code(
            min_positional_args, max_positional_args,
3504 3505
            has_fixed_positional_count, has_kw_only_args,
            all_args, argtuple_error_label, code)
3506 3507

        # --- optimised code when we do not receive any keyword arguments
3508
        if (self.num_required_kw_args and min_positional_args > 0) or min_positional_args == max_positional_args:
3509 3510 3511 3512 3513 3514 3515 3516 3517
            # Python raises arg tuple related errors first, so we must
            # check the length here
            if min_positional_args == max_positional_args and not self.star_arg:
                compare = '!='
            else:
                compare = '<'
            code.putln('} else if (PyTuple_GET_SIZE(%s) %s %d) {' % (
                    Naming.args_cname, compare, min_positional_args))
            code.put_goto(argtuple_error_label)
3518

3519 3520 3521 3522 3523 3524 3525 3526 3527
        if self.num_required_kw_args:
            # pure error case: keywords required but not passed
            if max_positional_args > min_positional_args and not self.star_arg:
                code.putln('} else if (PyTuple_GET_SIZE(%s) > %d) {' % (
                        Naming.args_cname, max_positional_args))
                code.put_goto(argtuple_error_label)
            code.putln('} else {')
            for i, arg in enumerate(kw_only_args):
                if not arg.default:
3528
                    pystring_cname = code.intern_identifier(arg.name)
3529
                    # required keyword-only argument missing
3530
                    code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' % (
Stefan Behnel's avatar
Stefan Behnel committed
3531
                            self.name,
3532
                            pystring_cname))
3533 3534
                    code.putln(code.error_goto(self.pos))
                    break
3535

3536
        else:
3537
            # optimised tuple unpacking code
3538
            code.putln('} else {')
3539 3540 3541 3542 3543
            if min_positional_args == max_positional_args:
                # parse the exact number of positional arguments from
                # the args tuple
                for i, arg in enumerate(positional_args):
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
3544
            else:
3545 3546 3547 3548 3549 3550 3551 3552
                # parse the positional arguments from the variable length
                # args tuple and reject illegal argument tuple sizes
                code.putln('switch (PyTuple_GET_SIZE(%s)) {' % Naming.args_cname)
                if self.star_arg:
                    code.putln('default:')
                reversed_args = list(enumerate(positional_args))[::-1]
                for i, arg in reversed_args:
                    if i >= min_positional_args-1:
3553
                        code.put('case %2d: ' % (i+1))
3554 3555 3556 3557 3558 3559 3560 3561 3562 3563 3564
                    code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (i, Naming.args_cname, i))
                if min_positional_args == 0:
                    code.put('case  0: ')
                code.putln('break;')
                if self.star_arg:
                    if min_positional_args:
                        for i in range(min_positional_args-1, -1, -1):
                            code.putln('case %2d:' % i)
                        code.put_goto(argtuple_error_label)
                else:
                    code.put('default: ')
3565
                    code.put_goto(argtuple_error_label)
3566 3567
                code.putln('}')

3568
        code.putln('}') # end of the conditional unpacking blocks
3569

3570 3571 3572
        # Convert arg values to their final type and assign them.
        # Also inject non-Python default arguments, which do cannot
        # live in the values[] array.
3573 3574
        for i, arg in enumerate(all_args):
            self.generate_arg_assignment(arg, "values[%d]" % i, code)
3575

3576
        code.putln('}') # end of the whole argument unpacking block
3577 3578 3579 3580

        if code.label_used(argtuple_error_label):
            code.put_goto(success_label)
            code.put_label(argtuple_error_label)
3581 3582
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3583
            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, PyTuple_GET_SIZE(%s)); ' % (
Stefan Behnel's avatar
Stefan Behnel committed
3584
                    self.name, has_fixed_positional_count,
3585 3586 3587 3588
                    min_positional_args, max_positional_args,
                    Naming.args_cname))
            code.putln(code.error_goto(self.pos))

3589 3590 3591 3592 3593 3594 3595 3596 3597 3598 3599 3600 3601 3602 3603 3604 3605 3606 3607 3608 3609 3610 3611 3612 3613 3614 3615 3616 3617 3618 3619 3620
    def generate_arg_assignment(self, arg, item, code):
        if arg.type.is_pyobject:
            # Python default arguments were already stored in 'item' at the very beginning
            if arg.is_generic:
                item = PyrexTypes.typecast(arg.type, PyrexTypes.py_object_type, item)
            entry = arg.entry
            code.putln("%s = %s;" % (entry.cname, item))
        else:
            func = arg.type.from_py_function
            if func:
                if arg.default:
                    # C-typed default arguments must be handled here
                    code.putln('if (%s) {' % item)
                rhs = "%s(%s)" % (func, item)
                if arg.type.is_enum:
                    rhs = arg.type.cast_code(rhs)
                code.putln("%s = %s; %s" % (
                    arg.entry.cname,
                    rhs,
                    code.error_goto_if(arg.type.error_condition(arg.entry.cname), arg.pos)))
                if arg.default:
                    code.putln('} else {')
                    code.putln(
                        "%s = %s;" % (
                            arg.entry.cname,
                            arg.calculate_default_value_code(code)))
                    if arg.type.is_memoryviewslice:
                        code.put_incref_memoryviewslice(arg.entry.cname,
                                                        have_gil=True)
                    code.putln('}')
            else:
                error(arg.pos, "Cannot convert Python object argument to type '%s'" % arg.type)
3621

3622
    def generate_stararg_init_code(self, max_positional_args, code):
3623
        if self.starstar_arg:
3624
            self.starstar_arg.entry.xdecref_cleanup = 0
Stefan Behnel's avatar
Stefan Behnel committed
3625 3626 3627 3628
            code.putln('%s = PyDict_New(); if (unlikely(!%s)) return %s;' % (
                    self.starstar_arg.entry.cname,
                    self.starstar_arg.entry.cname,
                    self.error_value()))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3629
            code.put_gotref(self.starstar_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3630 3631 3632 3633 3634
        if self.star_arg:
            self.star_arg.entry.xdecref_cleanup = 0
            code.putln('if (PyTuple_GET_SIZE(%s) > %d) {' % (
                    Naming.args_cname,
                    max_positional_args))
3635
            code.putln('%s = PyTuple_GetSlice(%s, %d, PyTuple_GET_SIZE(%s));' % (
Stefan Behnel's avatar
Stefan Behnel committed
3636 3637
                    self.star_arg.entry.cname, Naming.args_cname,
                    max_positional_args, Naming.args_cname))
3638
            code.putln("if (unlikely(!%s)) {" % self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3639
            if self.starstar_arg:
3640
                code.put_decref_clear(self.starstar_arg.entry.cname, py_object_type)
3641 3642 3643 3644
            code.put_finish_refcount_context()
            code.putln('return %s;' % self.error_value())
            code.putln('}')
            code.put_gotref(self.star_arg.entry.cname)
Stefan Behnel's avatar
Stefan Behnel committed
3645 3646 3647 3648
            code.putln('} else {')
            code.put("%s = %s; " % (self.star_arg.entry.cname, Naming.empty_tuple))
            code.put_incref(Naming.empty_tuple, py_object_type)
            code.putln('}')
3649

3650
    def generate_argument_values_setup_code(self, args, code):
3651
        max_args = len(args)
Stefan Behnel's avatar
Stefan Behnel committed
3652 3653
        # the 'values' array collects borrowed references to arguments
        # before doing any type coercion etc.
3654
        code.putln("PyObject* values[%d] = {%s};" % (
Stefan Behnel's avatar
Stefan Behnel committed
3655
            max_args, ','.join('0'*max_args)))
3656

3657
        if self.target.defaults_struct:
3658
            code.putln('%s *%s = __Pyx_CyFunction_Defaults(%s, %s);' % (
3659 3660
                self.target.defaults_struct, Naming.dynamic_args_cname,
                self.target.defaults_struct, Naming.self_cname))
3661

3662 3663
        # assign borrowed Python default values to the values array,
        # so that they can be overwritten by received arguments below
3664
        for i, arg in enumerate(args):
3665 3666 3667 3668
            if arg.default and arg.type.is_pyobject:
                default_value = arg.calculate_default_value_code(code)
                code.putln('values[%d] = %s;' % (i, arg.type.as_pyobject(default_value)))

3669 3670 3671 3672
    def generate_keyword_unpacking_code(self, min_positional_args, max_positional_args,
                                        has_fixed_positional_count, has_kw_only_args,
                                        all_args, argtuple_error_label, code):
        code.putln('Py_ssize_t kw_args;')
3673
        code.putln('const Py_ssize_t pos_args = PyTuple_GET_SIZE(%s);' % Naming.args_cname)
3674
        # copy the values from the args tuple and check that it's not too long
3675
        code.putln('switch (pos_args) {')
Stefan Behnel's avatar
Stefan Behnel committed
3676 3677
        if self.star_arg:
            code.putln('default:')
3678
        for i in range(max_positional_args-1, -1, -1):
3679
            code.put('case %2d: ' % (i+1))
3680 3681
            code.putln("values[%d] = PyTuple_GET_ITEM(%s, %d);" % (
                    i, Naming.args_cname, i))
3682
        code.putln('case  0: break;')
3683
        if not self.star_arg:
Stefan Behnel's avatar
Stefan Behnel committed
3684
            code.put('default: ') # more arguments than allowed
3685
            code.put_goto(argtuple_error_label)
3686 3687
        code.putln('}')

3688 3689 3690 3691 3692 3693 3694 3695 3696
        # The code above is very often (but not always) the same as
        # the optimised non-kwargs tuple unpacking code, so we keep
        # the code block above at the very top, before the following
        # 'external' PyDict_Size() call, to make it easy for the C
        # compiler to merge the two separate tuple unpacking
        # implementations into one when they turn out to be identical.

        # If we received kwargs, fill up the positional/required
        # arguments with values from the kw dict
3697
        code.putln('kw_args = PyDict_Size(%s);' % Naming.kwds_cname)
3698
        if self.num_required_args or max_positional_args > 0:
Stefan Behnel's avatar
Stefan Behnel committed
3699 3700 3701 3702
            last_required_arg = -1
            for i, arg in enumerate(all_args):
                if not arg.default:
                    last_required_arg = i
3703 3704
            if last_required_arg < max_positional_args:
                last_required_arg = max_positional_args-1
Stefan Behnel's avatar
Stefan Behnel committed
3705
            if max_positional_args > 0:
3706
                code.putln('switch (pos_args) {')
3707
            for i, arg in enumerate(all_args[:last_required_arg+1]):
Stefan Behnel's avatar
Stefan Behnel committed
3708
                if max_positional_args > 0 and i <= max_positional_args:
3709 3710 3711 3712
                    if self.star_arg and i == max_positional_args:
                        code.putln('default:')
                    else:
                        code.putln('case %2d:' % i)
3713
                pystring_cname = code.intern_identifier(arg.name)
3714
                if arg.default:
3715
                    if arg.kw_only:
3716
                        # optional kw-only args are handled separately below
3717
                        continue
3718
                    code.putln('if (kw_args > 0) {')
3719
                    # don't overwrite default argument
3720
                    code.putln('PyObject* value = PyDict_GetItem(%s, %s);' % (
3721
                        Naming.kwds_cname, pystring_cname))
3722
                    code.putln('if (value) { values[%d] = value; kw_args--; }' % i)
3723 3724
                    code.putln('}')
                else:
3725
                    code.putln('if (likely((values[%d] = PyDict_GetItem(%s, %s)) != 0)) kw_args--;' % (
3726
                        i, Naming.kwds_cname, pystring_cname))
3727 3728 3729 3730 3731 3732 3733 3734 3735 3736
                    if i < min_positional_args:
                        if i == 0:
                            # special case: we know arg 0 is missing
                            code.put('else ')
                            code.put_goto(argtuple_error_label)
                        else:
                            # print the correct number of values (args or
                            # kwargs) that were passed into positional
                            # arguments up to this point
                            code.putln('else {')
3737 3738
                            code.globalstate.use_utility_code(
                                UtilityCode.load_cached("RaiseArgTupleInvalid", "FunctionArguments.c"))
3739
                            code.put('__Pyx_RaiseArgtupleInvalid("%s", %d, %d, %d, %d); ' % (
Stefan Behnel's avatar
Stefan Behnel committed
3740
                                    self.name, has_fixed_positional_count,
3741 3742 3743 3744
                                    min_positional_args, max_positional_args, i))
                            code.putln(code.error_goto(self.pos))
                            code.putln('}')
                    elif arg.kw_only:
3745
                        code.putln('else {')
3746
                        code.put('__Pyx_RaiseKeywordRequired("%s", %s); ' %(
Stefan Behnel's avatar
Stefan Behnel committed
3747
                                self.name, pystring_cname))
3748 3749
                        code.putln(code.error_goto(self.pos))
                        code.putln('}')
Stefan Behnel's avatar
Stefan Behnel committed
3750 3751
            if max_positional_args > 0:
                code.putln('}')
3752

3753
        if has_kw_only_args:
3754
            # unpack optional keyword-only arguments separately because
3755
            # checking for interned strings in a dict is faster than iterating
3756
            self.generate_optional_kwonly_args_unpacking_code(all_args, code)
3757

3758
        code.putln('if (unlikely(kw_args > 0)) {')
3759 3760
        # non-positional/-required kw args left in dict: default args,
        # kw-only args, **kwargs or error
Stefan Behnel's avatar
Stefan Behnel committed
3761 3762 3763 3764 3765
        #
        # This is sort of a catch-all: except for checking required
        # arguments, this will always do the right thing for unpacking
        # keyword arguments, so that we can concentrate on optimising
        # common cases above.
3766 3767 3768
        if max_positional_args == 0:
            pos_arg_count = "0"
        elif self.star_arg:
3769 3770
            code.putln("const Py_ssize_t used_pos_args = (pos_args < %d) ? pos_args : %d;" % (
                    max_positional_args, max_positional_args))
Stefan Behnel's avatar
Stefan Behnel committed
3771 3772
            pos_arg_count = "used_pos_args"
        else:
3773
            pos_arg_count = "pos_args"
3774 3775
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("ParseKeywords", "FunctionArguments.c"))
Stefan Behnel's avatar
Stefan Behnel committed
3776 3777
        code.putln(
            'if (unlikely(__Pyx_ParseOptionalKeywords(%s, %s, %s, values, %s, "%s") < 0)) %s' % (
Stefan Behnel's avatar
Stefan Behnel committed
3778 3779 3780 3781
                Naming.kwds_cname,
                Naming.pykwdlist_cname,
                self.starstar_arg and self.starstar_arg.entry.cname or '0',
                pos_arg_count,
Stefan Behnel's avatar
Stefan Behnel committed
3782 3783
                self.name,
                code.error_goto(self.pos)))
3784
        code.putln('}')
3785

3786 3787 3788 3789 3790 3791 3792 3793 3794 3795 3796 3797 3798 3799 3800 3801 3802 3803 3804 3805 3806 3807 3808 3809 3810 3811 3812 3813 3814 3815 3816
    def generate_optional_kwonly_args_unpacking_code(self, all_args, code):
        optional_args = []
        first_optional_arg = -1
        for i, arg in enumerate(all_args):
            if not arg.kw_only or not arg.default:
                continue
            if not optional_args:
                first_optional_arg = i
            optional_args.append(arg.name)
        if optional_args:
            if len(optional_args) > 1:
                # if we receive more than the named kwargs, we either have **kwargs
                # (in which case we must iterate anyway) or it's an error (which we
                # also handle during iteration) => skip this part if there are more
                code.putln('if (kw_args > 0 && %s(kw_args <= %d)) {' % (
                    not self.starstar_arg and 'likely' or '',
                    len(optional_args)))
                code.putln('Py_ssize_t index;')
                # not unrolling the loop here reduces the C code overhead
                code.putln('for (index = %d; index < %d && kw_args > 0; index++) {' % (
                    first_optional_arg, first_optional_arg + len(optional_args)))
            else:
                code.putln('if (kw_args == 1) {')
                code.putln('const Py_ssize_t index = %d;' % first_optional_arg)
            code.putln('PyObject* value = PyDict_GetItem(%s, *%s[index]);' % (
                Naming.kwds_cname, Naming.pykwdlist_cname))
            code.putln('if (value) { values[index] = value; kw_args--; }')
            if len(optional_args) > 1:
                code.putln('}')
            code.putln('}')

William Stein's avatar
William Stein committed
3817
    def generate_argument_conversion_code(self, code):
3818 3819 3820
        # Generate code to convert arguments from signature type to
        # declared type, if needed.  Also copies signature arguments
        # into closure fields.
William Stein's avatar
William Stein committed
3821 3822 3823 3824 3825 3826 3827 3828 3829
        for arg in self.args:
            if arg.needs_conversion:
                self.generate_arg_conversion(arg, code)

    def generate_arg_conversion(self, arg, code):
        # Generate conversion code for one argument.
        old_type = arg.hdr_type
        new_type = arg.type
        if old_type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
3830 3831 3832 3833
            if arg.default:
                code.putln("if (%s) {" % arg.hdr_cname)
            else:
                code.putln("assert(%s); {" % arg.hdr_cname)
William Stein's avatar
William Stein committed
3834
            self.generate_arg_conversion_from_pyobject(arg, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
3835
            code.putln("}")
William Stein's avatar
William Stein committed
3836 3837 3838 3839 3840 3841 3842 3843
        elif new_type.is_pyobject:
            self.generate_arg_conversion_to_pyobject(arg, code)
        else:
            if new_type.assignable_from(old_type):
                code.putln(
                    "%s = %s;" % (arg.entry.cname, arg.hdr_cname))
            else:
                error(arg.pos,
3844
                    "Cannot convert 1 argument from '%s' to '%s'" %
William Stein's avatar
William Stein committed
3845
                        (old_type, new_type))
3846

William Stein's avatar
William Stein committed
3847 3848 3849
    def generate_arg_conversion_from_pyobject(self, arg, code):
        new_type = arg.type
        func = new_type.from_py_function
3850
        # copied from CoerceFromPyTypeNode
William Stein's avatar
William Stein committed
3851
        if func:
3852 3853 3854 3855 3856
            lhs = arg.entry.cname
            rhs = "%s(%s)" % (func, arg.hdr_cname)
            if new_type.is_enum:
                rhs = PyrexTypes.typecast(new_type, PyrexTypes.c_long_type, rhs)
            code.putln("%s = %s; %s" % (
3857
                lhs,
3858
                rhs,
3859
                code.error_goto_if(new_type.error_condition(arg.entry.cname), arg.pos)))
William Stein's avatar
William Stein committed
3860
        else:
3861 3862
            error(arg.pos,
                "Cannot convert Python object argument to type '%s'"
William Stein's avatar
William Stein committed
3863
                    % new_type)
3864

William Stein's avatar
William Stein committed
3865 3866 3867 3868
    def generate_arg_conversion_to_pyobject(self, arg, code):
        old_type = arg.hdr_type
        func = old_type.to_py_function
        if func:
Robert Bradshaw's avatar
Robert Bradshaw committed
3869
            code.putln("%s = %s(%s); %s" % (
William Stein's avatar
William Stein committed
3870 3871 3872
                arg.entry.cname,
                func,
                arg.hdr_cname,
Robert Bradshaw's avatar
Robert Bradshaw committed
3873
                code.error_goto_if_null(arg.entry.cname, arg.pos)))
3874
            code.put_var_gotref(arg.entry)
William Stein's avatar
William Stein committed
3875 3876 3877 3878 3879 3880 3881 3882 3883 3884 3885 3886
        else:
            error(arg.pos,
                "Cannot convert argument of type '%s' to Python object"
                    % old_type)

    def generate_argument_type_tests(self, code):
        # Generate type tests for args whose signature
        # type is PyObject * and whose declared type is
        # a subtype thereof.
        for arg in self.args:
            if arg.needs_type_test:
                self.generate_arg_type_test(arg, code)
3887 3888 3889
            elif not arg.accept_none and (arg.type.is_pyobject or
                                          arg.type.is_buffer or
                                          arg.type.is_memoryviewslice):
3890 3891
                self.generate_arg_none_check(arg, code)

William Stein's avatar
William Stein committed
3892
    def error_value(self):
3893
        return self.signature.error_value
3894

3895 3896

class GeneratorDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
3897
    # Generator function node that creates a new generator instance when called.
3898
    #
Stefan Behnel's avatar
Stefan Behnel committed
3899
    # gbody          GeneratorBodyDefNode   the function implementing the generator
3900 3901 3902 3903 3904
    #

    is_generator = True
    needs_closure = True

Stefan Behnel's avatar
Stefan Behnel committed
3905
    child_attrs = DefNode.child_attrs + ["gbody"]
3906

3907
    def __init__(self, pos, **kwargs):
3908
        # XXX: don't actually needs a body
3909 3910
        kwargs['body'] = StatListNode(pos, stats=[], is_terminator=True)
        super(GeneratorDefNode, self).__init__(pos, **kwargs)
3911 3912 3913 3914 3915 3916 3917 3918

    def analyse_declarations(self, env):
        super(GeneratorDefNode, self).analyse_declarations(env)
        self.gbody.local_scope = self.local_scope
        self.gbody.analyse_declarations(env)

    def generate_function_body(self, env, code):
        body_cname = self.gbody.entry.func_cname
3919 3920
        name = code.intern_identifier(self.name)
        qualname = code.intern_identifier(self.qualname)
3921

3922 3923
        code.putln('{')
        code.putln('__pyx_GeneratorObject *gen = __Pyx_Generator_New('
3924 3925
                   '(__pyx_generator_body_t) %s, (PyObject *) %s, %s, %s); %s' % (
                       body_cname, Naming.cur_scope_cname, name, qualname,
3926 3927
                       code.error_goto_if_null('gen', self.pos)))
        code.put_decref(Naming.cur_scope_cname, py_object_type)
3928
        if self.requires_classobj:
3929
            classobj_cname = 'gen->classobj'
3930 3931 3932 3933
            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                classobj_cname, Naming.self_cname))
            code.put_incref(classobj_cname, py_object_type)
            code.put_giveref(classobj_cname)
3934
        code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
3935
        code.putln('return (PyObject *) gen;')
3936
        code.putln('}')
3937 3938

    def generate_function_definitions(self, env, code):
Stefan Behnel's avatar
Stefan Behnel committed
3939
        env.use_utility_code(UtilityCode.load_cached("Generator", "Generator.c"))
3940

3941 3942 3943 3944 3945 3946
        self.gbody.generate_function_header(code, proto=True)
        super(GeneratorDefNode, self).generate_function_definitions(env, code)
        self.gbody.generate_function_definitions(env, code)


class GeneratorBodyDefNode(DefNode):
Stefan Behnel's avatar
Stefan Behnel committed
3947
    # Main code body of a generator implemented as a DefNode.
3948 3949 3950 3951
    #

    is_generator_body = True

3952
    def __init__(self, pos=None, name=None, body=None):
3953 3954 3955
        super(GeneratorBodyDefNode, self).__init__(
            pos=pos, body=body, name=name, doc=None,
            args=[], star_arg=None, starstar_arg=None)
3956

3957 3958
    def declare_generator_body(self, env):
        prefix = env.next_id(env.scope_prefix)
Vitja Makarov's avatar
Vitja Makarov committed
3959
        name = env.next_id('generator')
3960 3961 3962 3963
        cname = Naming.genbody_prefix + prefix + name
        entry = env.declare_var(None, py_object_type, self.pos,
                                cname=cname, visibility='private')
        entry.func_cname = cname
3964 3965 3966 3967 3968 3969
        entry.qualified_name = EncodedString(self.name)
        self.entry = entry

    def analyse_declarations(self, env):
        self.analyse_argument_types(env)
        self.declare_generator_body(env)
3970 3971

    def generate_function_header(self, code, proto=False):
3972
        header = "static PyObject *%s(__pyx_GeneratorObject *%s, PyObject *%s)" % (
3973
            self.entry.func_cname,
3974
            Naming.generator_cname,
3975 3976 3977 3978
            Naming.sent_value_cname)
        if proto:
            code.putln('%s; /* proto */' % header)
        else:
Stefan Behnel's avatar
Stefan Behnel committed
3979
            code.putln('%s /* generator body */\n{' % header)
3980 3981 3982 3983 3984 3985 3986 3987 3988 3989 3990 3991 3992 3993 3994 3995 3996

    def generate_function_definitions(self, env, code):
        lenv = self.local_scope

        # Generate closure function definitions
        self.body.generate_function_definitions(lenv, code)

        # Generate C code for header and body of function
        code.enter_cfunc_scope()
        code.return_from_error_cleanup_label = code.new_label()

        # ----- Top-level constants used by this function
        code.mark_pos(self.pos)
        self.generate_cached_builtins_decls(lenv, code)
        # ----- Function header
        code.putln("")
        self.generate_function_header(code)
3997
        closure_init_code = code.insertion_point()
3998 3999 4000 4001 4002 4003 4004 4005 4006 4007 4008 4009 4010 4011 4012 4013 4014
        # ----- Local variables
        code.putln("PyObject *%s = NULL;" % Naming.retval_cname)
        tempvardecl_code = code.insertion_point()
        code.put_declare_refcount_context()
        code.put_setup_refcount_context(self.entry.name)

        # ----- Resume switch point.
        code.funcstate.init_closure_temps(lenv.scope_class.type.scope)
        resume_code = code.insertion_point()
        first_run_label = code.new_label('first_run')
        code.use_label(first_run_label)
        code.put_label(first_run_label)
        code.putln('%s' %
                   (code.error_goto_if_null(Naming.sent_value_cname, self.pos)))

        # ----- Function body
        self.generate_function_body(env, code)
4015 4016 4017 4018 4019 4020
        # ----- Closure initialization
        if lenv.scope_class.type.scope.entries:
            closure_init_code.putln('%s = %s;' % (
                lenv.scope_class.type.declaration_code(Naming.cur_scope_cname),
                lenv.scope_class.type.cast_code('%s->closure' %
                                                Naming.generator_cname)))
4021 4022 4023 4024 4025

        code.mark_pos(self.pos)
        code.putln("")
        code.putln("/* function exit code */")

Stefan Behnel's avatar
Stefan Behnel committed
4026 4027
        # on normal generator termination, we do not take the exception propagation
        # path: no traceback info is required and not creating it is much faster
4028 4029
        if not self.body.is_terminator:
            code.putln('PyErr_SetNone(PyExc_StopIteration);')
4030 4031
        # ----- Error cleanup
        if code.error_label in code.labels_used:
4032 4033
            if not self.body.is_terminator:
                code.put_goto(code.return_label)
4034
            code.put_label(code.error_label)
4035 4036 4037 4038 4039
            if Future.generator_stop in env.global_scope().context.future_directives:
                # PEP 479: turn accidental StopIteration exceptions into a RuntimeError
                code.globalstate.use_utility_code(UtilityCode.load_cached("pep479", "Generator.c"))
                code.putln("if (unlikely(PyErr_ExceptionMatches(PyExc_StopIteration))) "
                           "__Pyx_Generator_Replace_StopIteration();")
4040 4041
            for cname, type in code.funcstate.all_managed_temps():
                code.put_xdecref(cname, type)
4042
            code.put_add_traceback(self.entry.qualified_name)
4043 4044 4045

        # ----- Non-error return cleanup
        code.put_label(code.return_label)
4046
        code.put_xdecref(Naming.retval_cname, py_object_type)
4047
        code.putln('%s->resume_label = -1;' % Naming.generator_cname)
4048 4049
        # clean up as early as possible to help breaking any reference cycles
        code.putln('__Pyx_Generator_clear((PyObject*)%s);' % Naming.generator_cname)
4050
        code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4051
        code.putln('return NULL;')
4052 4053 4054 4055 4056
        code.putln("}")

        # ----- Go back and insert temp variable declarations
        tempvardecl_code.put_temp_declarations(code.funcstate)
        # ----- Generator resume code
4057
        resume_code.putln("switch (%s->resume_label) {" % (
Stefan Behnel's avatar
Stefan Behnel committed
4058
                       Naming.generator_cname))
4059
        resume_code.putln("case 0: goto %s;" % first_run_label)
4060

4061 4062
        for i, label in code.yield_labels:
            resume_code.putln("case %d: goto %s;" % (i, label))
Stefan Behnel's avatar
Stefan Behnel committed
4063
        resume_code.putln("default: /* CPython raises the right error here */")
4064
        resume_code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
4065 4066
        resume_code.putln("return NULL;")
        resume_code.putln("}")
4067 4068 4069 4070

        code.exit_cfunc_scope()


4071 4072
class OverrideCheckNode(StatNode):
    # A Node for dispatching to the def method if it
4073
    # is overriden.
4074 4075 4076 4077 4078 4079
    #
    #  py_func
    #
    #  args
    #  func_temp
    #  body
4080

Robert Bradshaw's avatar
Robert Bradshaw committed
4081
    child_attrs = ['body']
4082

4083
    body = None
Robert Bradshaw's avatar
Robert Bradshaw committed
4084

4085 4086
    def analyse_expressions(self, env):
        self.args = env.arg_entries
4087 4088 4089 4090
        if self.py_func.is_module_scope:
            first_arg = 0
        else:
            first_arg = 1
4091
        from . import ExprNodes
4092
        self.func_node = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
Stefan Behnel's avatar
Stefan Behnel committed
4093 4094
        call_node = ExprNodes.SimpleCallNode(
            self.pos, function=self.func_node,
4095 4096
            args=[ ExprNodes.NameNode(self.pos, name=arg.name)
                   for arg in self.args[first_arg:] ])
4097
        self.body = ReturnStatNode(self.pos, value=call_node)
4098 4099
        self.body = self.body.analyse_expressions(env)
        return self
4100

4101
    def generate_execution_code(self, code):
4102
        interned_attr_cname = code.intern_identifier(self.py_func.entry.name)
4103
        # Check to see if we are an extension type
4104 4105 4106 4107
        if self.py_func.is_module_scope:
            self_arg = "((PyObject *)%s)" % Naming.module_cname
        else:
            self_arg = "((PyObject *)%s)" % self.args[0].cname
4108
        code.putln("/* Check if called by wrapper */")
4109
        code.putln("if (unlikely(%s)) ;" % Naming.skip_dispatch_cname)
Stefan Behnel's avatar
Stefan Behnel committed
4110
        code.putln("/* Check if overridden in Python */")
4111 4112 4113
        if self.py_func.is_module_scope:
            code.putln("else {")
        else:
4114
            code.putln("else if (unlikely(Py_TYPE(%s)->tp_dictoffset != 0)) {" % self_arg)
4115 4116
        func_node_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.func_node.set_cname(func_node_temp)
4117
        # need to get attribute manually--scope would return cdef method
4118 4119
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("PyObjectGetAttrStr", "ObjectHandling.c"))
4120
        err = code.error_goto_if_null(func_node_temp, self.pos)
4121
        code.putln("%s = __Pyx_PyObject_GetAttrStr(%s, %s); %s" % (
4122 4123 4124
            func_node_temp, self_arg, interned_attr_cname, err))
        code.put_gotref(func_node_temp)
        is_builtin_function_or_method = "PyCFunction_Check(%s)" % func_node_temp
4125
        is_overridden = "(PyCFunction_GET_FUNCTION(%s) != (PyCFunction)%s)" % (
4126
            func_node_temp, self.py_func.entry.func_cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
4127
        code.putln("if (!%s || %s) {" % (is_builtin_function_or_method, is_overridden))
4128 4129
        self.body.generate_execution_code(code)
        code.putln("}")
4130 4131
        code.put_decref_clear(func_node_temp, PyrexTypes.py_object_type)
        code.funcstate.release_temp(func_node_temp)
Robert Bradshaw's avatar
Robert Bradshaw committed
4132
        code.putln("}")
4133

Robert Bradshaw's avatar
Robert Bradshaw committed
4134 4135
class ClassDefNode(StatNode, BlockNode):
    pass
4136

Robert Bradshaw's avatar
Robert Bradshaw committed
4137
class PyClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4138 4139
    #  A Python class definition.
    #
Stefan Behnel's avatar
Stefan Behnel committed
4140
    #  name     EncodedString   Name of the class
William Stein's avatar
William Stein committed
4141 4142 4143 4144
    #  doc      string or None
    #  body     StatNode        Attribute definition code
    #  entry    Symtab.Entry
    #  scope    PyClassScope
4145
    #  decorators    [DecoratorNode]        list of decorators or None
William Stein's avatar
William Stein committed
4146 4147 4148
    #
    #  The following subnodes are constructed internally:
    #
4149
    #  dict     DictNode   Class dictionary or Py3 namespace
William Stein's avatar
William Stein committed
4150 4151
    #  classobj ClassNode  Class object
    #  target   NameNode   Variable to assign class object to
4152

4153
    child_attrs = ["body", "dict", "metaclass", "mkw", "bases", "class_result",
4154
                   "target", "class_cell", "decorators"]
4155
    decorators = None
4156
    class_result = None
4157 4158 4159
    is_py3_style_class = False  # Python3 style class (kwargs)
    metaclass = None
    mkw = None
4160

4161 4162
    def __init__(self, pos, name, bases, doc, body, decorators=None,
                 keyword_args=None, starstar_arg=None, force_py3_semantics=False):
William Stein's avatar
William Stein committed
4163 4164 4165 4166
        StatNode.__init__(self, pos)
        self.name = name
        self.doc = doc
        self.body = body
4167
        self.decorators = decorators
4168
        self.bases = bases
4169
        from . import ExprNodes
4170
        if self.doc and Options.docstrings:
4171
            doc = embed_position(self.pos, self.doc)
4172
            doc_node = ExprNodes.StringNode(pos, value=doc)
William Stein's avatar
William Stein committed
4173 4174
        else:
            doc_node = None
4175 4176

        allow_py2_metaclass = not force_py3_semantics
4177
        if keyword_args or starstar_arg:
4178 4179
            allow_py2_metaclass = False
            self.is_py3_style_class = True
4180 4181 4182 4183 4184 4185 4186 4187 4188 4189
            if keyword_args and not starstar_arg:
                for i, item in list(enumerate(keyword_args.key_value_pairs))[::-1]:
                    if item.key.value == 'metaclass':
                        if self.metaclass is not None:
                            error(item.pos, "keyword argument 'metaclass' passed multiple times")
                        # special case: we already know the metaclass,
                        # so we don't need to do the "build kwargs,
                        # find metaclass" dance at runtime
                        self.metaclass = item.value
                        del keyword_args.key_value_pairs[i]
4190
            if starstar_arg:
4191
                self.mkw = ExprNodes.KeywordArgsNode(
4192 4193 4194
                    pos, keyword_args=keyword_args and keyword_args.key_value_pairs or [],
                    starstar_arg=starstar_arg)
            elif keyword_args.key_value_pairs:
4195
                self.mkw = keyword_args
4196
            else:
4197 4198 4199
                assert self.metaclass is not None

        if force_py3_semantics or self.bases or self.mkw or self.metaclass:
4200
            if self.metaclass is None:
4201 4202 4203 4204 4205
                if starstar_arg:
                    # **kwargs may contain 'metaclass' arg
                    mkdict = self.mkw
                else:
                    mkdict = None
4206 4207 4208 4209 4210 4211 4212
                if (not mkdict and
                        self.bases.is_sequence_constructor and
                        not self.bases.args):
                    pass  # no base classes => no inherited metaclass
                else:
                    self.metaclass = ExprNodes.PyClassMetaclassNode(
                        pos, mkw=mkdict, bases=self.bases)
4213 4214 4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225
                needs_metaclass_calculation = False
            else:
                needs_metaclass_calculation = True

            self.dict = ExprNodes.PyClassNamespaceNode(
                pos, name=name, doc=doc_node,
                metaclass=self.metaclass, bases=self.bases, mkw=self.mkw)
            self.classobj = ExprNodes.Py3ClassNode(
                pos, name=name,
                bases=self.bases, dict=self.dict, doc=doc_node,
                metaclass=self.metaclass, mkw=self.mkw,
                calculate_metaclass=needs_metaclass_calculation,
                allow_py2_metaclass=allow_py2_metaclass)
4226
        else:
4227 4228 4229 4230 4231 4232 4233
            # no bases, no metaclass => old style class creation
            self.dict = ExprNodes.DictNode(pos, key_value_pairs=[])
            self.classobj = ExprNodes.ClassNode(
                pos, name=name,
                bases=bases, dict=self.dict, doc=doc_node)

        self.target = ExprNodes.NameNode(pos, name=name)
4234
        self.class_cell = ExprNodes.ClassCellInjectorNode(self.pos)
4235

4236 4237
    def as_cclass(self):
        """
4238
        Return this node as if it were declared as an extension class
4239
        """
4240
        if self.is_py3_style_class:
4241 4242
            error(self.classobj.pos, "Python3 style class could not be represented as C class")
            return
4243 4244 4245 4246 4247 4248 4249
        bases = self.classobj.bases.args
        if len(bases) == 0:
            base_class_name = None
            base_class_module = None
        elif len(bases) == 1:
            base = bases[0]
            path = []
4250
            from .ExprNodes import AttributeNode, NameNode
4251
            while isinstance(base, AttributeNode):
4252 4253
                path.insert(0, base.attribute)
                base = base.obj
4254
            if isinstance(base, NameNode):
4255 4256 4257 4258 4259 4260 4261 4262 4263 4264 4265
                path.insert(0, base.name)
                base_class_name = path[-1]
                if len(path) > 1:
                    base_class_module = u'.'.join(path[:-1])
                else:
                    base_class_module = None
            else:
                error(self.classobj.bases.args.pos, "Invalid base class")
        else:
            error(self.classobj.bases.args.pos, "C class may only have one base class")
            return None
4266 4267

        return CClassDefNode(self.pos,
4268 4269 4270 4271 4272
                             visibility = 'private',
                             module_name = None,
                             class_name = self.name,
                             base_class_module = base_class_module,
                             base_class_name = base_class_name,
4273
                             decorators = self.decorators,
4274 4275 4276
                             body = self.body,
                             in_pxd = False,
                             doc = self.doc)
4277

4278 4279
    def create_scope(self, env):
        genv = env
4280 4281
        while genv.is_py_class_scope or genv.is_c_class_scope:
            genv = genv.outer_scope
4282 4283
        cenv = self.scope = PyClassScope(name = self.name, outer_scope = genv)
        return cenv
4284

William Stein's avatar
William Stein committed
4285
    def analyse_declarations(self, env):
4286 4287
        class_result = self.classobj
        if self.decorators:
4288
            from .ExprNodes import SimpleCallNode
4289 4290 4291 4292 4293
            for decorator in self.decorators[::-1]:
                class_result = SimpleCallNode(
                    decorator.pos,
                    function = decorator.decorator,
                    args = [class_result])
4294
            self.decorators = None
4295 4296
        self.class_result = class_result
        self.class_result.analyse_declarations(env)
William Stein's avatar
William Stein committed
4297
        self.target.analyse_target_declaration(env)
4298
        cenv = self.create_scope(env)
4299
        cenv.directives = env.directives
4300 4301
        cenv.class_obj_cname = self.target.entry.cname
        self.body.analyse_declarations(cenv)
4302

William Stein's avatar
William Stein committed
4303
    def analyse_expressions(self, env):
4304
        if self.bases:
4305
            self.bases = self.bases.analyse_expressions(env)
4306
        if self.metaclass:
4307
            self.metaclass = self.metaclass.analyse_expressions(env)
4308
        if self.mkw:
4309 4310 4311
            self.mkw = self.mkw.analyse_expressions(env)
        self.dict = self.dict.analyse_expressions(env)
        self.class_result = self.class_result.analyse_expressions(env)
William Stein's avatar
William Stein committed
4312
        genv = env.global_scope()
4313
        cenv = self.scope
4314
        self.body = self.body.analyse_expressions(cenv)
4315
        self.target.analyse_target_expression(env, self.classobj)
4316 4317
        self.class_cell = self.class_cell.analyse_expressions(cenv)
        return self
4318

4319
    def generate_function_definitions(self, env, code):
4320
        self.generate_lambda_definitions(self.scope, code)
4321
        self.body.generate_function_definitions(self.scope, code)
4322

William Stein's avatar
William Stein committed
4323
    def generate_execution_code(self, code):
4324 4325
        code.pyclass_stack.append(self)
        cenv = self.scope
4326
        if self.bases:
4327
            self.bases.generate_evaluation_code(code)
4328
        if self.mkw:
4329
            self.mkw.generate_evaluation_code(code)
4330
        if self.metaclass:
4331
            self.metaclass.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
4332
        self.dict.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4333
        cenv.namespace_cname = cenv.class_obj_cname = self.dict.result()
4334
        self.class_cell.generate_evaluation_code(code)
Vitja Makarov's avatar
Vitja Makarov committed
4335
        self.body.generate_execution_code(code)
4336
        self.class_result.generate_evaluation_code(code)
4337 4338 4339
        self.class_cell.generate_injection_code(
            code, self.class_result.result())
        self.class_cell.generate_disposal_code(code)
4340
        cenv.namespace_cname = cenv.class_obj_cname = self.classobj.result()
4341
        self.target.generate_assignment_code(self.class_result, code)
William Stein's avatar
William Stein committed
4342
        self.dict.generate_disposal_code(code)
4343
        self.dict.free_temps(code)
4344
        if self.metaclass:
4345 4346
            self.metaclass.generate_disposal_code(code)
            self.metaclass.free_temps(code)
4347 4348 4349 4350
        if self.mkw:
            self.mkw.generate_disposal_code(code)
            self.mkw.free_temps(code)
        if self.bases:
4351 4352
            self.bases.generate_disposal_code(code)
            self.bases.free_temps(code)
4353
        code.pyclass_stack.pop()
William Stein's avatar
William Stein committed
4354

Robert Bradshaw's avatar
Robert Bradshaw committed
4355
class CClassDefNode(ClassDefNode):
William Stein's avatar
William Stein committed
4356 4357 4358 4359
    #  An extension type definition.
    #
    #  visibility         'private' or 'public' or 'extern'
    #  typedef_flag       boolean
Stefan Behnel's avatar
Stefan Behnel committed
4360
    #  api                boolean
William Stein's avatar
William Stein committed
4361 4362 4363 4364 4365 4366 4367 4368
    #  module_name        string or None    For import of extern type objects
    #  class_name         string            Unqualified name of class
    #  as_name            string or None    Name to declare as in this scope
    #  base_class_module  string or None    Module containing the base class
    #  base_class_name    string or None    Name of the base class
    #  objstruct_name     string or None    Specified C name of object struct
    #  typeobj_name       string or None    Specified C name of type object
    #  in_pxd             boolean           Is in a .pxd file
4369
    #  decorators         [DecoratorNode]   list of decorators or None
William Stein's avatar
William Stein committed
4370 4371 4372 4373
    #  doc                string or None
    #  body               StatNode or None
    #  entry              Symtab.Entry
    #  base_type          PyExtensionType or None
4374 4375
    #  buffer_defaults_node DictNode or None Declares defaults for a buffer
    #  buffer_defaults_pos
4376

4377
    child_attrs = ["body"]
4378 4379
    buffer_defaults_node = None
    buffer_defaults_pos = None
4380 4381 4382 4383
    typedef_flag = False
    api = False
    objstruct_name = None
    typeobj_name = None
4384
    decorators = None
4385
    shadow = False
4386

Robert Bradshaw's avatar
Robert Bradshaw committed
4387 4388
    def buffer_defaults(self, env):
        if not hasattr(self, '_buffer_defaults'):
4389
            from . import Buffer
Robert Bradshaw's avatar
Robert Bradshaw committed
4390 4391 4392 4393 4394 4395 4396 4397 4398
            if self.buffer_defaults_node:
                self._buffer_defaults = Buffer.analyse_buffer_options(
                    self.buffer_defaults_pos,
                    env, [], self.buffer_defaults_node,
                    need_complete=False)
            else:
                self._buffer_defaults = None
        return self._buffer_defaults

4399 4400 4401 4402 4403 4404 4405 4406
    def declare(self, env):
        if self.module_name and self.visibility != 'extern':
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return None
        else:
            home_scope = env
4407

4408 4409 4410 4411 4412 4413 4414 4415 4416 4417 4418 4419
        self.entry = home_scope.declare_c_class(
            name = self.class_name,
            pos = self.pos,
            defining = 0,
            implementing = 0,
            module_name = self.module_name,
            base_type = None,
            objstruct_cname = self.objstruct_name,
            typeobj_cname = self.typeobj_name,
            visibility = self.visibility,
            typedef_flag = self.typedef_flag,
            api = self.api,
Robert Bradshaw's avatar
Robert Bradshaw committed
4420
            buffer_defaults = self.buffer_defaults(env),
4421 4422
            shadow = self.shadow)

William Stein's avatar
William Stein committed
4423 4424 4425 4426
    def analyse_declarations(self, env):
        #print "CClassDefNode.analyse_declarations:", self.class_name
        #print "...visibility =", self.visibility
        #print "...module_name =", self.module_name
4427

William Stein's avatar
William Stein committed
4428 4429 4430
        if env.in_cinclude and not self.objstruct_name:
            error(self.pos, "Object struct name specification required for "
                "C class defined in 'extern from' block")
4431 4432 4433
        if self.decorators:
            error(self.pos,
                  "Decorators not allowed on cdef classes (used on type '%s')" % self.class_name)
William Stein's avatar
William Stein committed
4434
        self.base_type = None
4435 4436
        # Now that module imports are cached, we need to
        # import the modules for extern classes.
4437 4438 4439 4440 4441 4442 4443
        if self.module_name:
            self.module = None
            for module in env.cimported_modules:
                if module.name == self.module_name:
                    self.module = module
            if self.module is None:
                self.module = ModuleScope(self.module_name, None, env.context)
4444
                self.module.has_extern_class = 1
Robert Bradshaw's avatar
Robert Bradshaw committed
4445
                env.add_imported_module(self.module)
4446

William Stein's avatar
William Stein committed
4447 4448 4449 4450 4451
        if self.base_class_name:
            if self.base_class_module:
                base_class_scope = env.find_module(self.base_class_module, self.pos)
            else:
                base_class_scope = env
4452 4453 4454 4455 4456 4457
            if self.base_class_name == 'object':
                # extension classes are special and don't need to inherit from object
                if base_class_scope is None or base_class_scope.lookup('object') is None:
                    self.base_class_name = None
                    self.base_class_module = None
                    base_class_scope = None
William Stein's avatar
William Stein committed
4458 4459 4460 4461 4462
            if base_class_scope:
                base_class_entry = base_class_scope.find(self.base_class_name, self.pos)
                if base_class_entry:
                    if not base_class_entry.is_type:
                        error(self.pos, "'%s' is not a type name" % self.base_class_name)
4463
                    elif not base_class_entry.type.is_extension_type and \
Stefan Behnel's avatar
Stefan Behnel committed
4464
                             not (base_class_entry.type.is_builtin_type and
4465
                                  base_class_entry.type.objstruct_cname):
William Stein's avatar
William Stein committed
4466 4467
                        error(self.pos, "'%s' is not an extension type" % self.base_class_name)
                    elif not base_class_entry.type.is_complete():
4468 4469 4470
                        error(self.pos, "Base class '%s' of type '%s' is incomplete" % (
                            self.base_class_name, self.class_name))
                    elif base_class_entry.type.scope and base_class_entry.type.scope.directives and \
4471
                             base_class_entry.type.is_final_type:
4472 4473
                        error(self.pos, "Base class '%s' of type '%s' is final" % (
                            self.base_class_name, self.class_name))
4474 4475 4476 4477
                    elif base_class_entry.type.is_builtin_type and \
                             base_class_entry.type.name in ('tuple', 'str', 'bytes'):
                        error(self.pos, "inheritance from PyVarObject types like '%s' is not currently supported"
                              % base_class_entry.type.name)
William Stein's avatar
William Stein committed
4478 4479
                    else:
                        self.base_type = base_class_entry.type
4480
                if env.directives.get('freelist', 0) > 0:
Stefan Behnel's avatar
Stefan Behnel committed
4481
                    warning(self.pos, "freelists cannot be used on subtypes, only the base class can manage them", 1)
4482

William Stein's avatar
William Stein committed
4483
        has_body = self.body is not None
4484 4485 4486 4487 4488 4489
        if has_body and self.base_type and not self.base_type.scope:
            # To properly initialize inherited attributes, the base type must
            # be analysed before this type.
            self.base_type.defered_declarations.append(lambda : self.analyse_declarations(env))
            return

4490
        if self.module_name and self.visibility != 'extern':
4491 4492 4493 4494 4495 4496
            module_path = self.module_name.split(".")
            home_scope = env.find_imported_module(module_path, self.pos)
            if not home_scope:
                return
        else:
            home_scope = env
4497 4498

        if self.visibility == 'extern':
4499 4500 4501
            if (self.module_name == '__builtin__' and
                self.class_name in Builtin.builtin_types and
                env.qualified_name[:8] != 'cpython.'): # allow overloaded names for cimporting from cpython
4502
                warning(self.pos, "%s already a builtin Cython type" % self.class_name, 1)
4503

4504
        self.entry = home_scope.declare_c_class(
4505
            name = self.class_name,
William Stein's avatar
William Stein committed
4506 4507 4508 4509 4510 4511 4512 4513
            pos = self.pos,
            defining = has_body and self.in_pxd,
            implementing = has_body and not self.in_pxd,
            module_name = self.module_name,
            base_type = self.base_type,
            objstruct_cname = self.objstruct_name,
            typeobj_cname = self.typeobj_name,
            visibility = self.visibility,
Stefan Behnel's avatar
Stefan Behnel committed
4514
            typedef_flag = self.typedef_flag,
4515
            api = self.api,
Robert Bradshaw's avatar
Robert Bradshaw committed
4516
            buffer_defaults = self.buffer_defaults(env),
4517
            shadow = self.shadow)
4518

4519 4520
        if self.shadow:
            home_scope.lookup(self.class_name).as_variable = self.entry
4521
        if home_scope is not env and self.visibility == 'extern':
4522
            env.add_imported_entry(self.class_name, self.entry, self.pos)
4523
        self.scope = scope = self.entry.type.scope
4524 4525
        if scope is not None:
            scope.directives = env.directives
4526

4527
        if self.doc and Options.docstrings:
4528
            scope.doc = embed_position(self.pos, self.doc)
4529

William Stein's avatar
William Stein committed
4530 4531 4532 4533 4534 4535 4536
        if has_body:
            self.body.analyse_declarations(scope)
            if self.in_pxd:
                scope.defined = 1
            else:
                scope.implemented = 1
        env.allocate_vtable_names(self.entry)
4537

4538 4539 4540
        for thunk in self.entry.type.defered_declarations:
            thunk()

William Stein's avatar
William Stein committed
4541 4542
    def analyse_expressions(self, env):
        if self.body:
Robert Bradshaw's avatar
Robert Bradshaw committed
4543
            scope = self.entry.type.scope
4544 4545
            self.body = self.body.analyse_expressions(scope)
        return self
4546

4547
    def generate_function_definitions(self, env, code):
William Stein's avatar
William Stein committed
4548
        if self.body:
4549 4550
            self.generate_lambda_definitions(self.scope, code)
            self.body.generate_function_definitions(self.scope, code)
4551

William Stein's avatar
William Stein committed
4552 4553 4554 4555 4556
    def generate_execution_code(self, code):
        # This is needed to generate evaluation code for
        # default values of method arguments.
        if self.body:
            self.body.generate_execution_code(code)
4557

4558 4559 4560
    def annotate(self, code):
        if self.body:
            self.body.annotate(code)
William Stein's avatar
William Stein committed
4561 4562 4563 4564 4565 4566


class PropertyNode(StatNode):
    #  Definition of a property in an extension type.
    #
    #  name   string
4567
    #  doc    EncodedString or None    Doc string
4568
    #  entry  Symtab.Entry
William Stein's avatar
William Stein committed
4569
    #  body   StatListNode
4570

4571 4572
    child_attrs = ["body"]

William Stein's avatar
William Stein committed
4573
    def analyse_declarations(self, env):
4574 4575 4576
        self.entry = env.declare_property(self.name, self.doc, self.pos)
        self.entry.scope.directives = env.directives
        self.body.analyse_declarations(self.entry.scope)
4577

William Stein's avatar
William Stein committed
4578
    def analyse_expressions(self, env):
4579 4580
        self.body = self.body.analyse_expressions(env)
        return self
4581

4582 4583
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
William Stein's avatar
William Stein committed
4584 4585 4586 4587

    def generate_execution_code(self, code):
        pass

4588 4589 4590
    def annotate(self, code):
        self.body.annotate(code)

William Stein's avatar
William Stein committed
4591 4592 4593 4594 4595

class GlobalNode(StatNode):
    # Global variable declaration.
    #
    # names    [string]
4596

4597 4598
    child_attrs = []

William Stein's avatar
William Stein committed
4599 4600 4601 4602 4603
    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_global(name, self.pos)

    def analyse_expressions(self, env):
4604
        return self
4605

William Stein's avatar
William Stein committed
4606 4607 4608 4609
    def generate_execution_code(self, code):
        pass


4610 4611 4612 4613 4614 4615 4616 4617 4618 4619 4620 4621
class NonlocalNode(StatNode):
    # Nonlocal variable declaration via the 'nonlocal' keyword.
    #
    # names    [string]

    child_attrs = []

    def analyse_declarations(self, env):
        for name in self.names:
            env.declare_nonlocal(name, self.pos)

    def analyse_expressions(self, env):
4622
        return self
4623 4624 4625 4626 4627

    def generate_execution_code(self, code):
        pass


William Stein's avatar
William Stein committed
4628 4629 4630 4631
class ExprStatNode(StatNode):
    #  Expression used as a statement.
    #
    #  expr   ExprNode
4632 4633

    child_attrs = ["expr"]
4634

Robert Bradshaw's avatar
Robert Bradshaw committed
4635
    def analyse_declarations(self, env):
4636
        from . import ExprNodes
Robert Bradshaw's avatar
Robert Bradshaw committed
4637
        if isinstance(self.expr, ExprNodes.GeneralCallNode):
4638
            func = self.expr.function.as_cython_attribute()
Robert Bradshaw's avatar
Robert Bradshaw committed
4639 4640 4641 4642 4643 4644 4645 4646 4647 4648 4649
            if func == u'declare':
                args, kwds = self.expr.explicit_args_kwds()
                if len(args):
                    error(self.expr.pos, "Variable names must be specified.")
                for var, type_node in kwds.key_value_pairs:
                    type = type_node.analyse_as_type(env)
                    if type is None:
                        error(type_node.pos, "Unknown type")
                    else:
                        env.declare_var(var.value, type, var.pos, is_cdef = True)
                self.__class__ = PassStatNode
4650

William Stein's avatar
William Stein committed
4651
    def analyse_expressions(self, env):
4652
        self.expr.result_is_used = False # hint that .result() may safely be left empty
4653 4654
        self.expr = self.expr.analyse_expressions(env)
        return self
4655

4656
    def nogil_check(self, env):
4657
        if self.expr.type.is_pyobject and self.expr.is_temp:
4658 4659 4660 4661
            self.gil_error()

    gil_message = "Discarding owned Python object"

William Stein's avatar
William Stein committed
4662 4663
    def generate_execution_code(self, code):
        self.expr.generate_evaluation_code(code)
4664 4665
        if not self.expr.is_temp and self.expr.result():
            code.putln("%s;" % self.expr.result())
William Stein's avatar
William Stein committed
4666
        self.expr.generate_disposal_code(code)
4667
        self.expr.free_temps(code)
William Stein's avatar
William Stein committed
4668

4669 4670 4671
    def generate_function_definitions(self, env, code):
        self.expr.generate_function_definitions(env, code)

4672 4673 4674
    def annotate(self, code):
        self.expr.annotate(code)

William Stein's avatar
William Stein committed
4675 4676 4677 4678 4679 4680 4681 4682 4683 4684 4685

class AssignmentNode(StatNode):
    #  Abstract base class for assignment nodes.
    #
    #  The analyse_expressions and generate_execution_code
    #  phases of assignments are split into two sub-phases
    #  each, to enable all the right hand sides of a
    #  parallel assignment to be evaluated before assigning
    #  to any of the left hand sides.

    def analyse_expressions(self, env):
4686
        node = self.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
4687
        if isinstance(node, AssignmentNode) and not isinstance(node, ParallelAssignmentNode):
4688
            if node.rhs.type.is_ptr and node.rhs.is_ephemeral():
4689 4690
                error(self.pos, "Storing unsafe C derivative of temporary Python reference")
        return node
4691

4692 4693 4694
#       def analyse_expressions(self, env):
#           self.analyse_expressions_1(env)
#           self.analyse_expressions_2(env)
William Stein's avatar
William Stein committed
4695 4696 4697 4698

    def generate_execution_code(self, code):
        self.generate_rhs_evaluation_code(code)
        self.generate_assignment_code(code)
4699

William Stein's avatar
William Stein committed
4700 4701 4702 4703 4704 4705 4706 4707

class SingleAssignmentNode(AssignmentNode):
    #  The simplest case:
    #
    #    a = b
    #
    #  lhs      ExprNode      Left hand side
    #  rhs      ExprNode      Right hand side
4708
    #  first    bool          Is this guaranteed the first assignment to lhs?
4709

4710
    child_attrs = ["lhs", "rhs"]
4711
    first = False
4712
    declaration_only = False
William Stein's avatar
William Stein committed
4713 4714

    def analyse_declarations(self, env):
4715
        from . import ExprNodes
4716

4717 4718
        # handle declarations of the form x = cython.foo()
        if isinstance(self.rhs, ExprNodes.CallNode):
4719
            func_name = self.rhs.function.as_cython_attribute()
4720 4721
            if func_name:
                args, kwds = self.rhs.explicit_args_kwds()
4722

4723
                if func_name in ['declare', 'typedef']:
Robert Bradshaw's avatar
Robert Bradshaw committed
4724
                    if len(args) > 2 or kwds is not None:
4725
                        error(self.rhs.pos, "Can only declare one type at a time.")
4726
                        return
4727

4728 4729 4730 4731 4732 4733 4734 4735 4736 4737 4738 4739 4740 4741 4742
                    type = args[0].analyse_as_type(env)
                    if type is None:
                        error(args[0].pos, "Unknown type")
                        return
                    lhs = self.lhs
                    if func_name == 'declare':
                        if isinstance(lhs, ExprNodes.NameNode):
                            vars = [(lhs.name, lhs.pos)]
                        elif isinstance(lhs, ExprNodes.TupleNode):
                            vars = [(var.name, var.pos) for var in lhs.args]
                        else:
                            error(lhs.pos, "Invalid declaration")
                            return
                        for var, pos in vars:
                            env.declare_var(var, type, pos, is_cdef = True)
Robert Bradshaw's avatar
Robert Bradshaw committed
4743 4744 4745 4746 4747
                        if len(args) == 2:
                            # we have a value
                            self.rhs = args[1]
                        else:
                            self.declaration_only = True
4748
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
4749
                        self.declaration_only = True
4750 4751
                        if not isinstance(lhs, ExprNodes.NameNode):
                            error(lhs.pos, "Invalid declaration.")
4752
                        env.declare_typedef(lhs.name, type, self.pos, visibility='private')
4753

4754 4755 4756
                elif func_name in ['struct', 'union']:
                    self.declaration_only = True
                    if len(args) > 0 or kwds is None:
4757
                        error(self.rhs.pos, "Struct or union members must be given by name.")
4758 4759 4760 4761 4762 4763 4764 4765 4766 4767 4768 4769 4770 4771 4772 4773 4774
                        return
                    members = []
                    for member, type_node in kwds.key_value_pairs:
                        type = type_node.analyse_as_type(env)
                        if type is None:
                            error(type_node.pos, "Unknown type")
                        else:
                            members.append((member.value, type, member.pos))
                    if len(members) < len(kwds.key_value_pairs):
                        return
                    if not isinstance(self.lhs, ExprNodes.NameNode):
                        error(self.lhs.pos, "Invalid declaration.")
                    name = self.lhs.name
                    scope = StructOrUnionScope(name)
                    env.declare_struct_or_union(name, func_name, scope, False, self.rhs.pos)
                    for member, type, pos in members:
                        scope.declare_var(member, type, pos)
4775

Mark Florisson's avatar
Mark Florisson committed
4776 4777 4778 4779 4780 4781 4782 4783
                elif func_name == 'fused_type':
                    # dtype = cython.fused_type(...)
                    self.declaration_only = True
                    if kwds:
                        error(self.rhs.function.pos,
                              "fused_type does not take keyword arguments")

                    fusednode = FusedTypeNode(self.rhs.pos,
4784
                                              name=self.lhs.name, types=args)
Mark Florisson's avatar
Mark Florisson committed
4785 4786
                    fusednode.analyse_declarations(env)

4787 4788 4789 4790
        if self.declaration_only:
            return
        else:
            self.lhs.analyse_target_declaration(env)
4791

4792 4793
    def analyse_types(self, env, use_temp=0):
        from . import ExprNodes
4794

4795
        self.rhs = self.rhs.analyse_types(env)
4796

4797 4798 4799
        unrolled_assignment = self.unroll_rhs(env)
        if unrolled_assignment:
            return unrolled_assignment
Robert Bradshaw's avatar
Robert Bradshaw committed
4800

4801
        self.lhs = self.lhs.analyse_target_types(env)
4802
        self.lhs.gil_assignment_check(env)
4803 4804 4805
        unrolled_assignment = self.unroll_lhs(env)
        if unrolled_assignment:
            return unrolled_assignment
4806 4807 4808 4809 4810

        if self.lhs.memslice_broadcast or self.rhs.memslice_broadcast:
            self.lhs.memslice_broadcast = True
            self.rhs.memslice_broadcast = True

4811 4812 4813 4814
        if (self.lhs.is_subscript and not self.rhs.type.is_memoryviewslice and
                (self.lhs.memslice_slice or self.lhs.is_memslice_copy) and
                (self.lhs.type.dtype.assignable_from(self.rhs.type) or
                 self.rhs.type.is_pyobject)):
4815 4816 4817
            # scalar slice assignment
            self.lhs.is_memslice_scalar_assignment = True
            dtype = self.lhs.type.dtype
4818
        elif self.lhs.type.is_array:
4819
            if not isinstance(self.lhs, ExprNodes.SliceIndexNode):
4820 4821 4822 4823 4824
                # cannot assign to C array, only to its full slice
                self.lhs = ExprNodes.SliceIndexNode(
                    self.lhs.pos, base=self.lhs, start=None, stop=None)
                self.lhs = self.lhs.analyse_target_types(env)
            dtype = self.lhs.type
4825 4826 4827
        else:
            dtype = self.lhs.type

4828 4829 4830 4831
        rhs = self.rhs.coerce_to(dtype, env)
        if use_temp or rhs.is_attribute or (
                not rhs.is_name and not rhs.is_literal and
                rhs.type.is_pyobject):
4832
            # things like (cdef) attribute access are not safe (traverses pointers)
4833 4834 4835 4836
            rhs = rhs.coerce_to_temp(env)
        elif rhs.type.is_pyobject:
            rhs = rhs.coerce_to_simple(env)
        self.rhs = rhs
4837
        return self
4838

4839 4840
    def unroll(self, node, target_size, env):
        from . import ExprNodes, UtilNodes
4841 4842 4843 4844

        base = node
        start_node = stop_node = step_node = check_node = None

4845
        if node.type.is_ctuple:
4846
            slice_size = node.type.size
4847

4848 4849 4850
        elif node.type.is_ptr or node.type.is_array:
            while isinstance(node, ExprNodes.SliceIndexNode) and not (node.start or node.stop):
                base = node = node.base
4851 4852 4853
            if isinstance(node, ExprNodes.SliceIndexNode):
                base = node.base
                start_node = node.start
4854 4855
                if start_node:
                    start_node = start_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
4856
                stop_node = node.stop
4857 4858 4859
                if stop_node:
                    stop_node = stop_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
                else:
4860
                    if node.type.is_array and node.type.size:
4861 4862 4863 4864
                        stop_node = ExprNodes.IntNode(
                            self.pos, value=str(node.type.size),
                            constant_result=(node.type.size if isinstance(node.type.size, (int, long))
                                             else ExprNodes.constant_value_not_set))
4865 4866 4867
                    else:
                        error(self.pos, "C array iteration requires known end index")
                        return
4868
                step_node = None  #node.step
4869 4870
                if step_node:
                    step_node = step_node.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
4871

4872 4873 4874 4875
                # TODO: Factor out SliceIndexNode.generate_slice_guard_code() for use here.
                def get_const(node, none_value):
                    if node is None:
                        return none_value
4876
                    elif node.has_constant_result():
4877 4878
                        return node.constant_result
                    else:
4879 4880
                        raise ValueError("Not a constant.")

4881 4882 4883 4884 4885
                try:
                    slice_size = (get_const(stop_node, None) - get_const(start_node, 0)) / get_const(step_node, 1)
                except ValueError:
                    error(self.pos, "C array assignment currently requires known endpoints")
                    return
4886 4887 4888 4889 4890

            elif node.type.is_array:
                slice_size = node.type.size
                if not isinstance(slice_size, (int, long)):
                    return  # might still work when coercing to Python
4891 4892 4893 4894 4895 4896
            else:
                return

        else:
            return

4897 4898 4899 4900 4901
        if slice_size != target_size:
            error(self.pos, "Assignment to/from slice of wrong length, expected %s, got %s" % (
                slice_size, target_size))
            return

4902
        items = []
4903 4904 4905
        base = UtilNodes.LetRefNode(base)
        refs = [base]
        if start_node and not start_node.is_literal:
4906 4907
            start_node = UtilNodes.LetRefNode(start_node)
            refs.append(start_node)
4908
        if stop_node and not stop_node.is_literal:
4909 4910
            stop_node = UtilNodes.LetRefNode(stop_node)
            refs.append(stop_node)
4911
        if step_node and not step_node.is_literal:
4912 4913
            step_node = UtilNodes.LetRefNode(step_node)
            refs.append(step_node)
4914

4915
        for ix in range(target_size):
4916
            ix_node = ExprNodes.IntNode(self.pos, value=str(ix), constant_result=ix, type=PyrexTypes.c_py_ssize_t_type)
4917
            if step_node is not None:
4918 4919 4920 4921 4922
                if step_node.has_constant_result():
                    step_value = ix_node.constant_result * step_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(step_value), constant_result=step_value)
                else:
                    ix_node = ExprNodes.MulNode(self.pos, operator='*', operand1=step_node, operand2=ix_node)
4923
            if start_node is not None:
4924 4925 4926 4927 4928 4929 4930
                if start_node.has_constant_result() and ix_node.has_constant_result():
                    index_value = ix_node.constant_result + start_node.constant_result
                    ix_node = ExprNodes.IntNode(self.pos, value=str(index_value), constant_result=index_value)
                else:
                    ix_node = ExprNodes.AddNode(
                        self.pos, operator='+', operand1=start_node, operand2=ix_node)
            items.append(ExprNodes.IndexNode(self.pos, base=base, index=ix_node.analyse_types(env)))
4931 4932
        return check_node, refs, items

4933
    def unroll_assignments(self, refs, check_node, lhs_list, rhs_list, env):
4934
        from . import UtilNodes
4935
        assignments = []
4936
        for lhs, rhs in zip(lhs_list, rhs_list):
4937
            assignments.append(SingleAssignmentNode(self.pos, lhs=lhs, rhs=rhs, first=self.first))
4938 4939 4940
        all = ParallelAssignmentNode(pos=self.pos, stats=assignments).analyse_expressions(env)
        if check_node:
            all = StatListNode(pos=self.pos, stats=[check_node, all])
4941
        for ref in refs[::-1]:
4942 4943 4944
            all = UtilNodes.LetNode(ref, all)
        return all

4945
    def unroll_rhs(self, env):
4946
        from . import ExprNodes
4947 4948
        if not isinstance(self.lhs, ExprNodes.TupleNode):
            return
4949 4950
        if any(arg.is_starred for arg in self.lhs.args):
            return
4951 4952 4953 4954 4955 4956 4957 4958 4959 4960 4961 4962 4963 4964 4965 4966 4967 4968 4969 4970 4971

        unrolled = self.unroll(self.rhs, len(self.lhs.args), env)
        if not unrolled:
            return
        check_node, refs, rhs = unrolled
        return self.unroll_assignments(refs, check_node, self.lhs.args, rhs, env)

    def unroll_lhs(self, env):
        if self.lhs.type.is_ctuple:
            # Handled directly.
            return
        from . import ExprNodes, UtilNodes
        if not isinstance(self.rhs, ExprNodes.TupleNode):
            return

        unrolled = self.unroll(self.lhs, len(self.rhs.args), env)
        if not unrolled:
            return
        check_node, refs, lhs = unrolled
        return self.unroll_assignments(refs, check_node, lhs, self.rhs.args, env)

William Stein's avatar
William Stein committed
4972 4973
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
4974

William Stein's avatar
William Stein committed
4975 4976 4977
    def generate_assignment_code(self, code):
        self.lhs.generate_assignment_code(self.rhs, code)

4978 4979 4980
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

4981 4982 4983 4984
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

William Stein's avatar
William Stein committed
4985 4986 4987 4988 4989 4990 4991 4992 4993 4994 4995

class CascadedAssignmentNode(AssignmentNode):
    #  An assignment with multiple left hand sides:
    #
    #    a = b = c
    #
    #  lhs_list   [ExprNode]   Left hand sides
    #  rhs        ExprNode     Right hand sides
    #
    #  Used internally:
    #
4996 4997
    #  coerced_values     [ExprNode]   RHS coerced to all distinct LHS types
    #  cloned_values      [ExprNode]   cloned RHS value for each LHS
4998

4999 5000
    child_attrs = ["lhs_list", "rhs", "coerced_values", "cloned_values"]
    cloned_values = None
5001
    coerced_values = None
5002

William Stein's avatar
William Stein committed
5003 5004 5005
    def analyse_declarations(self, env):
        for lhs in self.lhs_list:
            lhs.analyse_target_declaration(env)
5006

5007
    def analyse_types(self, env, use_temp=0):
5008
        from .ExprNodes import CloneNode, ProxyNode
5009

5010
        # collect distinct types used on the LHS
5011 5012 5013 5014 5015 5016
        lhs_types = set()
        for lhs in self.lhs_list:
            lhs.analyse_target_types(env)
            lhs.gil_assignment_check(env)
            lhs_types.add(lhs.type)

5017
        rhs = self.rhs.analyse_types(env)
5018 5019 5020 5021 5022 5023
        if len(lhs_types) == 1:
            # common special case: only one type needed on the LHS => coerce only once
            rhs = rhs.coerce_to(lhs_types.pop(), env)

        if not rhs.is_name and not rhs.is_literal and (
                use_temp or rhs.is_attribute or rhs.type.is_pyobject):
5024
            rhs = rhs.coerce_to_temp(env)
5025
        else:
5026
            rhs = rhs.coerce_to_simple(env)
5027 5028
        self.rhs = ProxyNode(rhs) if rhs.is_temp else rhs

5029
        # clone RHS and coerce it to all distinct LHS types
5030 5031 5032 5033 5034 5035 5036
        self.coerced_values = []
        coerced_values = {}
        for lhs in self.lhs_list:
            if lhs.type not in coerced_values and lhs.type != rhs.type:
                rhs = CloneNode(self.rhs).coerce_to(lhs.type, env)
                self.coerced_values.append(rhs)
                coerced_values[lhs.type] = rhs
5037

5038 5039
        # clone coerced values for all LHS assignments
        self.cloned_values = []
William Stein's avatar
William Stein committed
5040
        for lhs in self.lhs_list:
5041
            rhs = coerced_values.get(lhs.type, self.rhs)
5042
            self.cloned_values.append(CloneNode(rhs))
5043
        return self
5044

William Stein's avatar
William Stein committed
5045 5046
    def generate_rhs_evaluation_code(self, code):
        self.rhs.generate_evaluation_code(code)
5047

William Stein's avatar
William Stein committed
5048
    def generate_assignment_code(self, code):
5049
        # prepare all coercions
5050 5051
        for rhs in self.coerced_values:
            rhs.generate_evaluation_code(code)
5052 5053
        # assign clones to LHS
        for lhs, rhs in zip(self.lhs_list, self.cloned_values):
William Stein's avatar
William Stein committed
5054 5055
            rhs.generate_evaluation_code(code)
            lhs.generate_assignment_code(rhs, code)
5056 5057 5058 5059
        # dispose of coerced values and original RHS
        for rhs_value in self.coerced_values:
            rhs_value.generate_disposal_code(code)
            rhs_value.free_temps(code)
William Stein's avatar
William Stein committed
5060
        self.rhs.generate_disposal_code(code)
5061
        self.rhs.free_temps(code)
William Stein's avatar
William Stein committed
5062

5063 5064 5065
    def generate_function_definitions(self, env, code):
        self.rhs.generate_function_definitions(env, code)

5066
    def annotate(self, code):
5067 5068
        for rhs in self.coerced_values:
            rhs.annotate(code)
5069
        for lhs, rhs in zip(self.lhs_list, self.cloned_values):
5070 5071
            lhs.annotate(code)
            rhs.annotate(code)
5072
        self.rhs.annotate(code)
5073

5074

William Stein's avatar
William Stein committed
5075 5076 5077 5078 5079 5080 5081 5082 5083 5084 5085 5086 5087
class ParallelAssignmentNode(AssignmentNode):
    #  A combined packing/unpacking assignment:
    #
    #    a, b, c =  d, e, f
    #
    #  This has been rearranged by the parser into
    #
    #    a = d ; b = e ; c = f
    #
    #  but we must evaluate all the right hand sides
    #  before assigning to any of the left hand sides.
    #
    #  stats     [AssignmentNode]   The constituent assignments
5088

5089 5090
    child_attrs = ["stats"]

William Stein's avatar
William Stein committed
5091 5092 5093
    def analyse_declarations(self, env):
        for stat in self.stats:
            stat.analyse_declarations(env)
5094

William Stein's avatar
William Stein committed
5095
    def analyse_expressions(self, env):
5096 5097 5098
        self.stats = [ stat.analyse_types(env, use_temp = 1)
                       for stat in self.stats ]
        return self
5099

Robert Bradshaw's avatar
Robert Bradshaw committed
5100 5101 5102 5103 5104
#    def analyse_expressions(self, env):
#        for stat in self.stats:
#            stat.analyse_expressions_1(env, use_temp = 1)
#        for stat in self.stats:
#            stat.analyse_expressions_2(env)
5105

William Stein's avatar
William Stein committed
5106 5107 5108 5109 5110 5111
    def generate_execution_code(self, code):
        for stat in self.stats:
            stat.generate_rhs_evaluation_code(code)
        for stat in self.stats:
            stat.generate_assignment_code(code)

5112 5113 5114 5115
    def generate_function_definitions(self, env, code):
        for stat in self.stats:
            stat.generate_function_definitions(env, code)

5116 5117 5118 5119 5120
    def annotate(self, code):
        for stat in self.stats:
            stat.annotate(code)


5121
class InPlaceAssignmentNode(AssignmentNode):
Craig Citro's avatar
Craig Citro committed
5122
    #  An in place arithmetic operand:
5123 5124 5125 5126 5127 5128 5129
    #
    #    a += b
    #    a -= b
    #    ...
    #
    #  lhs      ExprNode      Left hand side
    #  rhs      ExprNode      Right hand side
Stefan Behnel's avatar
Stefan Behnel committed
5130
    #  operator char          one of "+-*/%^&|"
5131
    #
5132 5133 5134 5135 5136 5137 5138
    #  This code is a bit tricky because in order to obey Python
    #  semantics the sub-expressions (e.g. indices) of the lhs must
    #  not be evaluated twice. So we must re-use the values calculated
    #  in evaluation phase for the assignment phase as well.
    #  Fortunately, the type of the lhs node is fairly constrained
    #  (it must be a NameNode, AttributeNode, or IndexNode).

5139
    child_attrs = ["lhs", "rhs"]
5140

5141 5142
    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)
5143

5144
    def analyse_types(self, env):
5145 5146
        self.rhs = self.rhs.analyse_types(env)
        self.lhs = self.lhs.analyse_target_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5147

5148
        # When assigning to a fully indexed buffer or memoryview, coerce the rhs
Stefan Behnel's avatar
Stefan Behnel committed
5149
        if (self.lhs.is_subscript and
5150 5151
                (self.lhs.memslice_index or self.lhs.is_buffer_access)):
            self.rhs = self.rhs.coerce_to(self.lhs.type, env)
5152 5153 5154
        elif self.lhs.type.is_string and self.operator in '+-':
            # use pointer arithmetic for char* LHS instead of string concat
            self.rhs = self.rhs.coerce_to(PyrexTypes.c_py_ssize_t_type, env)
5155
        return self
5156

5157
    def generate_execution_code(self, code):
5158 5159 5160 5161 5162 5163 5164
        self.rhs.generate_evaluation_code(code)
        self.lhs.generate_subexpr_evaluation_code(code)
        c_op = self.operator
        if c_op == "//":
            c_op = "/"
        elif c_op == "**":
            error(self.pos, "No C inplace power operator")
Stefan Behnel's avatar
Stefan Behnel committed
5165
        if self.lhs.is_subscript and self.lhs.is_buffer_access:
5166 5167
            if self.lhs.type.is_pyobject:
                error(self.pos, "In-place operators not allowed on object buffers in this release.")
5168 5169
            if (c_op in ('/', '%') and self.lhs.type.is_int
                and not code.globalstate.directives['cdivision']):
Robert Bradshaw's avatar
Robert Bradshaw committed
5170
                error(self.pos, "In-place non-c divide operators not allowed on int buffers.")
5171 5172 5173 5174 5175 5176 5177 5178 5179 5180
            self.lhs.generate_buffer_setitem_code(self.rhs, code, c_op)
        else:
            # C++
            # TODO: make sure overload is declared
            code.putln("%s %s= %s;" % (self.lhs.result(), c_op, self.rhs.result()))
        self.lhs.generate_subexpr_disposal_code(code)
        self.lhs.free_subexpr_temps(code)
        self.rhs.generate_disposal_code(code)
        self.rhs.free_temps(code)

5181 5182 5183
    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)
5184

5185
    def create_binop_node(self):
5186
        from . import ExprNodes
5187
        return ExprNodes.binop_node(self.pos, self.operator, self.lhs, self.rhs)
5188

William Stein's avatar
William Stein committed
5189 5190 5191 5192

class PrintStatNode(StatNode):
    #  print statement
    #
5193
    #  arg_tuple         TupleNode
5194
    #  stream            ExprNode or None (stdout)
5195
    #  append_newline    boolean
5196

5197
    child_attrs = ["arg_tuple", "stream"]
5198

William Stein's avatar
William Stein committed
5199
    def analyse_expressions(self, env):
5200
        if self.stream:
5201 5202 5203 5204
            stream = self.stream.analyse_expressions(env)
            self.stream = stream.coerce_to_pyobject(env)
        arg_tuple = self.arg_tuple.analyse_expressions(env)
        self.arg_tuple = arg_tuple.coerce_to_pyobject(env)
5205
        env.use_utility_code(printing_utility_code)
5206 5207
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            env.use_utility_code(printing_one_utility_code)
5208
        return self
5209

5210
    nogil_check = Node.gil_error
5211
    gil_message = "Python print statement"
5212

William Stein's avatar
William Stein committed
5213
    def generate_execution_code(self, code):
5214 5215 5216 5217 5218
        if self.stream:
            self.stream.generate_evaluation_code(code)
            stream_result = self.stream.py_result()
        else:
            stream_result = '0'
5219 5220 5221
        if len(self.arg_tuple.args) == 1 and self.append_newline:
            arg = self.arg_tuple.args[0]
            arg.generate_evaluation_code(code)
5222

5223
            code.putln(
5224 5225
                "if (__Pyx_PrintOne(%s, %s) < 0) %s" % (
                    stream_result,
5226 5227 5228 5229 5230 5231 5232
                    arg.py_result(),
                    code.error_goto(self.pos)))
            arg.generate_disposal_code(code)
            arg.free_temps(code)
        else:
            self.arg_tuple.generate_evaluation_code(code)
            code.putln(
5233 5234
                "if (__Pyx_Print(%s, %s, %d) < 0) %s" % (
                    stream_result,
5235 5236 5237 5238 5239
                    self.arg_tuple.py_result(),
                    self.append_newline,
                    code.error_goto(self.pos)))
            self.arg_tuple.generate_disposal_code(code)
            self.arg_tuple.free_temps(code)
5240

5241 5242 5243 5244
        if self.stream:
            self.stream.generate_disposal_code(code)
            self.stream.free_temps(code)

5245
    def generate_function_definitions(self, env, code):
5246 5247
        if self.stream:
            self.stream.generate_function_definitions(env, code)
5248
        self.arg_tuple.generate_function_definitions(env, code)
5249

5250
    def annotate(self, code):
5251 5252
        if self.stream:
            self.stream.annotate(code)
5253
        self.arg_tuple.annotate(code)
William Stein's avatar
William Stein committed
5254 5255


5256 5257 5258 5259 5260 5261 5262 5263 5264
class ExecStatNode(StatNode):
    #  exec statement
    #
    #  args     [ExprNode]

    child_attrs = ["args"]

    def analyse_expressions(self, env):
        for i, arg in enumerate(self.args):
5265
            arg = arg.analyse_expressions(env)
5266 5267 5268
            arg = arg.coerce_to_pyobject(env)
            self.args[i] = arg
        env.use_utility_code(Builtin.pyexec_utility_code)
5269
        return self
5270

5271
    nogil_check = Node.gil_error
5272 5273 5274 5275 5276 5277 5278 5279
    gil_message = "Python exec statement"

    def generate_execution_code(self, code):
        args = []
        for arg in self.args:
            arg.generate_evaluation_code(code)
            args.append( arg.py_result() )
        args = tuple(args + ['0', '0'][:3-len(args)])
5280
        temp_result = code.funcstate.allocate_temp(PyrexTypes.py_object_type, manage_ref=True)
Stefan Behnel's avatar
Stefan Behnel committed
5281
        code.putln("%s = __Pyx_PyExec3(%s, %s, %s);" % (
5282
                (temp_result,) + args))
5283 5284
        for arg in self.args:
            arg.generate_disposal_code(code)
5285
            arg.free_temps(code)
5286
        code.putln(
5287 5288 5289 5290
            code.error_goto_if_null(temp_result, self.pos))
        code.put_gotref(temp_result)
        code.put_decref_clear(temp_result, py_object_type)
        code.funcstate.release_temp(temp_result)
5291 5292 5293 5294 5295 5296

    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)


William Stein's avatar
William Stein committed
5297 5298 5299 5300
class DelStatNode(StatNode):
    #  del statement
    #
    #  args     [ExprNode]
5301

5302
    child_attrs = ["args"]
5303
    ignore_nonexisting = False
5304

William Stein's avatar
William Stein committed
5305 5306 5307
    def analyse_declarations(self, env):
        for arg in self.args:
            arg.analyse_target_declaration(env)
5308

William Stein's avatar
William Stein committed
5309
    def analyse_expressions(self, env):
5310 5311
        for i, arg in enumerate(self.args):
            arg = self.args[i] = arg.analyse_target_expression(env, None)
5312 5313
            if arg.type.is_pyobject or (arg.is_name and
                                        arg.type.is_memoryviewslice):
5314 5315
                if arg.is_name and arg.entry.is_cglobal:
                    error(arg.pos, "Deletion of global C variable")
Robert Bradshaw's avatar
Robert Bradshaw committed
5316
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
Robert Bradshaw's avatar
Robert Bradshaw committed
5317
                self.cpp_check(env)
5318
            elif arg.type.is_cpp_class:
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5319
                error(arg.pos, "Deletion of non-heap C++ object")
5320 5321
            elif arg.is_subscript and arg.base.type is Builtin.bytearray_type:
                pass  # del ba[i]
5322
            else:
Robert Bradshaw's avatar
Robert Bradshaw committed
5323
                error(arg.pos, "Deletion of non-Python, non-C++ object")
5324
            #arg.release_target_temp(env)
5325
        return self
5326

5327
    def nogil_check(self, env):
5328 5329
        for arg in self.args:
            if arg.type.is_pyobject:
5330
                self.gil_error()
5331

5332 5333
    gil_message = "Deleting Python object"

William Stein's avatar
William Stein committed
5334 5335
    def generate_execution_code(self, code):
        for arg in self.args:
5336 5337 5338
            if (arg.type.is_pyobject or
                    arg.type.is_memoryviewslice or
                    arg.is_subscript and arg.base.type is Builtin.bytearray_type):
5339 5340
                arg.generate_deletion_code(
                    code, ignore_nonexisting=self.ignore_nonexisting)
5341
            elif arg.type.is_ptr and arg.type.base_type.is_cpp_class:
Robert Bradshaw's avatar
Robert Bradshaw committed
5342
                arg.generate_result_code(code)
Robert Bradshaw's avatar
merge  
Robert Bradshaw committed
5343
                code.putln("delete %s;" % arg.result())
William Stein's avatar
William Stein committed
5344 5345
            # else error reported earlier

5346 5347 5348 5349
    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)

William Stein's avatar
William Stein committed
5350 5351 5352

class PassStatNode(StatNode):
    #  pass statement
5353 5354

    child_attrs = []
5355

William Stein's avatar
William Stein committed
5356
    def analyse_expressions(self, env):
5357
        return self
5358

William Stein's avatar
William Stein committed
5359 5360 5361 5362
    def generate_execution_code(self, code):
        pass


5363 5364 5365 5366 5367 5368 5369 5370 5371
class IndirectionNode(StatListNode):
    """
    This adds an indirection so that the node can be shared and a subtree can
    be removed at any time by clearing self.stats.
    """

    def __init__(self, stats):
        super(IndirectionNode, self).__init__(stats[0].pos, stats=stats)

William Stein's avatar
William Stein committed
5372 5373
class BreakStatNode(StatNode):

5374
    child_attrs = []
5375
    is_terminator = True
5376

William Stein's avatar
William Stein committed
5377
    def analyse_expressions(self, env):
5378
        return self
5379

William Stein's avatar
William Stein committed
5380 5381 5382 5383
    def generate_execution_code(self, code):
        if not code.break_label:
            error(self.pos, "break statement not inside loop")
        else:
5384
            code.put_goto(code.break_label)
William Stein's avatar
William Stein committed
5385 5386 5387 5388


class ContinueStatNode(StatNode):

5389
    child_attrs = []
5390
    is_terminator = True
5391

William Stein's avatar
William Stein committed
5392
    def analyse_expressions(self, env):
5393
        return self
5394

William Stein's avatar
William Stein committed
5395
    def generate_execution_code(self, code):
5396
        if code.funcstate.in_try_finally:
William Stein's avatar
William Stein committed
5397 5398 5399 5400
            error(self.pos, "continue statement inside try of try...finally")
        elif not code.continue_label:
            error(self.pos, "continue statement not inside loop")
        else:
5401
            code.put_goto(code.continue_label)
William Stein's avatar
William Stein committed
5402 5403 5404 5405 5406 5407 5408


class ReturnStatNode(StatNode):
    #  return statement
    #
    #  value         ExprNode or None
    #  return_type   PyrexType
5409
    #  in_generator  return inside of generator => raise StopIteration
5410

5411
    child_attrs = ["value"]
5412
    is_terminator = True
5413
    in_generator = False
5414

5415 5416 5417
    # Whether we are in a parallel section
    in_parallel = False

William Stein's avatar
William Stein committed
5418 5419 5420 5421 5422
    def analyse_expressions(self, env):
        return_type = env.return_type
        self.return_type = return_type
        if not return_type:
            error(self.pos, "Return not inside a function body")
5423
            return self
William Stein's avatar
William Stein committed
5424
        if self.value:
5425
            self.value = self.value.analyse_types(env)
William Stein's avatar
William Stein committed
5426
            if return_type.is_void or return_type.is_returncode:
5427
                error(self.value.pos,
William Stein's avatar
William Stein committed
5428 5429 5430 5431 5432 5433 5434 5435
                    "Return with value in void function")
            else:
                self.value = self.value.coerce_to(env.return_type, env)
        else:
            if (not return_type.is_void
                and not return_type.is_pyobject
                and not return_type.is_returncode):
                    error(self.pos, "Return value required")
5436
        return self
5437

5438
    def nogil_check(self, env):
5439
        if self.return_type.is_pyobject:
5440
            self.gil_error()
5441 5442 5443

    gil_message = "Returning Python object"

William Stein's avatar
William Stein committed
5444
    def generate_execution_code(self, code):
5445
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
5446 5447 5448
        if not self.return_type:
            # error reported earlier
            return
5449 5450 5451
        if self.return_type.is_pyobject:
            code.put_xdecref(Naming.retval_cname,
                             self.return_type)
5452

William Stein's avatar
William Stein committed
5453 5454
        if self.value:
            self.value.generate_evaluation_code(code)
5455
            if self.return_type.is_memoryviewslice:
5456
                from . import MemoryView
5457 5458 5459 5460 5461
                MemoryView.put_acquire_memoryviewslice(
                        lhs_cname=Naming.retval_cname,
                        lhs_type=self.return_type,
                        lhs_pos=self.value.pos,
                        rhs=self.value,
5462 5463
                        code=code,
                        have_gil=self.in_nogil_context)
5464 5465 5466 5467 5468 5469 5470
            elif self.in_generator:
                # return value == raise StopIteration(value), but uncatchable
                code.putln(
                    "%s = NULL; PyErr_SetObject(PyExc_StopIteration, %s);" % (
                        Naming.retval_cname,
                        self.value.result_as(self.return_type)))
                self.value.generate_disposal_code(code)
5471 5472 5473 5474 5475 5476
            else:
                self.value.make_owned_reference(code)
                code.putln(
                    "%s = %s;" % (
                        Naming.retval_cname,
                        self.value.result_as(self.return_type)))
5477
            self.value.generate_post_assignment_code(code)
5478
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
5479 5480 5481 5482
        else:
            if self.return_type.is_pyobject:
                code.put_init_to_py_none(Naming.retval_cname, self.return_type)
            elif self.return_type.is_returncode:
5483 5484
                self.put_return(code, self.return_type.default_value)

5485
        for cname, type in code.funcstate.temps_holding_reference():
5486
            code.put_decref_clear(cname, type)
5487

5488
        code.put_goto(code.return_label)
5489

5490 5491 5492 5493 5494
    def put_return(self, code, value):
        if self.in_parallel:
            code.putln_openmp("#pragma omp critical(__pyx_returning)")
        code.putln("%s = %s;" % (Naming.retval_cname, value))

5495 5496 5497
    def generate_function_definitions(self, env, code):
        if self.value is not None:
            self.value.generate_function_definitions(env, code)
5498

5499 5500 5501
    def annotate(self, code):
        if self.value:
            self.value.annotate(code)
William Stein's avatar
William Stein committed
5502 5503 5504 5505 5506 5507 5508 5509


class RaiseStatNode(StatNode):
    #  raise statement
    #
    #  exc_type    ExprNode or None
    #  exc_value   ExprNode or None
    #  exc_tb      ExprNode or None
Haoyu Bai's avatar
Haoyu Bai committed
5510
    #  cause       ExprNode or None
5511

Haoyu Bai's avatar
Haoyu Bai committed
5512
    child_attrs = ["exc_type", "exc_value", "exc_tb", "cause"]
5513
    is_terminator = True
5514

William Stein's avatar
William Stein committed
5515 5516
    def analyse_expressions(self, env):
        if self.exc_type:
5517 5518
            exc_type = self.exc_type.analyse_types(env)
            self.exc_type = exc_type.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
5519
        if self.exc_value:
5520 5521
            exc_value = self.exc_value.analyse_types(env)
            self.exc_value = exc_value.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
5522
        if self.exc_tb:
5523 5524
            exc_tb = self.exc_tb.analyse_types(env)
            self.exc_tb = exc_tb.coerce_to_pyobject(env)
Haoyu Bai's avatar
Haoyu Bai committed
5525
        if self.cause:
5526 5527
            cause = self.cause.analyse_types(env)
            self.cause = cause.coerce_to_pyobject(env)
5528 5529 5530 5531
        # special cases for builtin exceptions
        self.builtin_exc_name = None
        if self.exc_type and not self.exc_value and not self.exc_tb:
            exc = self.exc_type
5532
            from . import ExprNodes
Robert Bradshaw's avatar
Robert Bradshaw committed
5533 5534
            if (isinstance(exc, ExprNodes.SimpleCallNode) and
                not (exc.args or (exc.arg_tuple is not None and
5535
                                  exc.arg_tuple.args))):
5536 5537 5538 5539 5540
                exc = exc.function # extract the exception type
            if exc.is_name and exc.entry.is_builtin:
                self.builtin_exc_name = exc.name
                if self.builtin_exc_name == 'MemoryError':
                    self.exc_type = None # has a separate implementation
5541
        return self
5542

5543
    nogil_check = Node.gil_error
5544 5545
    gil_message = "Raising exception"

William Stein's avatar
William Stein committed
5546
    def generate_execution_code(self, code):
5547 5548 5549 5550
        if self.builtin_exc_name == 'MemoryError':
            code.putln('PyErr_NoMemory(); %s' % code.error_goto(self.pos))
            return

William Stein's avatar
William Stein committed
5551 5552 5553 5554
        if self.exc_type:
            self.exc_type.generate_evaluation_code(code)
            type_code = self.exc_type.py_result()
        else:
Stefan Behnel's avatar
Stefan Behnel committed
5555
            type_code = "0"
William Stein's avatar
William Stein committed
5556 5557 5558 5559 5560 5561 5562 5563 5564 5565
        if self.exc_value:
            self.exc_value.generate_evaluation_code(code)
            value_code = self.exc_value.py_result()
        else:
            value_code = "0"
        if self.exc_tb:
            self.exc_tb.generate_evaluation_code(code)
            tb_code = self.exc_tb.py_result()
        else:
            tb_code = "0"
Haoyu Bai's avatar
Haoyu Bai committed
5566 5567 5568 5569 5570
        if self.cause:
            self.cause.generate_evaluation_code(code)
            cause_code = self.cause.py_result()
        else:
            cause_code = "0"
5571
        code.globalstate.use_utility_code(raise_utility_code)
5572
        code.putln(
Haoyu Bai's avatar
Haoyu Bai committed
5573
            "__Pyx_Raise(%s, %s, %s, %s);" % (
5574 5575
                type_code,
                value_code,
Haoyu Bai's avatar
Haoyu Bai committed
5576 5577 5578
                tb_code,
                cause_code))
        for obj in (self.exc_type, self.exc_value, self.exc_tb, self.cause):
5579 5580 5581
            if obj:
                obj.generate_disposal_code(code)
                obj.free_temps(code)
William Stein's avatar
William Stein committed
5582 5583 5584
        code.putln(
            code.error_goto(self.pos))

5585 5586 5587 5588 5589 5590 5591
    def generate_function_definitions(self, env, code):
        if self.exc_type is not None:
            self.exc_type.generate_function_definitions(env, code)
        if self.exc_value is not None:
            self.exc_value.generate_function_definitions(env, code)
        if self.exc_tb is not None:
            self.exc_tb.generate_function_definitions(env, code)
Haoyu Bai's avatar
Haoyu Bai committed
5592 5593
        if self.cause is not None:
            self.cause.generate_function_definitions(env, code)
5594

5595 5596 5597 5598 5599 5600 5601
    def annotate(self, code):
        if self.exc_type:
            self.exc_type.annotate(code)
        if self.exc_value:
            self.exc_value.annotate(code)
        if self.exc_tb:
            self.exc_tb.annotate(code)
Haoyu Bai's avatar
Haoyu Bai committed
5602 5603
        if self.cause:
            self.cause.annotate(code)
5604

William Stein's avatar
William Stein committed
5605

5606 5607
class ReraiseStatNode(StatNode):

5608
    child_attrs = []
5609
    is_terminator = True
5610

5611
    def analyse_expressions(self, env):
5612
        return self
5613

5614
    nogil_check = Node.gil_error
5615 5616
    gil_message = "Raising exception"

5617
    def generate_execution_code(self, code):
5618
        vars = code.funcstate.exc_vars
5619
        if vars:
5620
            code.globalstate.use_utility_code(restore_exception_utility_code)
5621 5622 5623 5624
            code.put_giveref(vars[0])
            code.put_giveref(vars[1])
            # fresh exceptions may not have a traceback yet (-> finally!)
            code.put_xgiveref(vars[2])
5625 5626 5627 5628
            code.putln("__Pyx_ErrRestore(%s, %s, %s);" % tuple(vars))
            for varname in vars:
                code.put("%s = 0; " % varname)
            code.putln()
5629 5630
            code.putln(code.error_goto(self.pos))
        else:
5631 5632 5633
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ReRaiseException", "Exceptions.c"))
            code.putln("__Pyx_ReraiseException(); %s" % code.error_goto(self.pos))
5634

William Stein's avatar
William Stein committed
5635 5636 5637 5638 5639
class AssertStatNode(StatNode):
    #  assert statement
    #
    #  cond    ExprNode
    #  value   ExprNode or None
5640

5641 5642
    child_attrs = ["cond", "value"]

William Stein's avatar
William Stein committed
5643 5644 5645
    def analyse_expressions(self, env):
        self.cond = self.cond.analyse_boolean_expression(env)
        if self.value:
5646
            value = self.value.analyse_types(env)
5647 5648
            if value.type is Builtin.tuple_type or not value.type.is_builtin_type:
                # prevent tuple values from being interpreted as argument value tuples
5649
                from .ExprNodes import TupleNode
5650
                value = TupleNode(value.pos, args=[value], slow=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
5651
                self.value = value.analyse_types(env, skip_children=True).coerce_to_pyobject(env)
5652 5653
            else:
                self.value = value.coerce_to_pyobject(env)
5654
        return self
5655

5656
    nogil_check = Node.gil_error
5657
    gil_message = "Raising exception"
5658

William Stein's avatar
William Stein committed
5659
    def generate_execution_code(self, code):
5660
        code.putln("#ifndef CYTHON_WITHOUT_ASSERTIONS")
5661
        code.putln("if (unlikely(!Py_OptimizeFlag)) {")
William Stein's avatar
William Stein committed
5662 5663
        self.cond.generate_evaluation_code(code)
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
5664
            "if (unlikely(!%s)) {" %
5665
                self.cond.result())
William Stein's avatar
William Stein committed
5666
        if self.value:
5667
            self.value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
5668 5669 5670
            code.putln(
                "PyErr_SetObject(PyExc_AssertionError, %s);" %
                    self.value.py_result())
5671
            self.value.generate_disposal_code(code)
5672
            self.value.free_temps(code)
William Stein's avatar
William Stein committed
5673 5674 5675 5676 5677 5678 5679 5680
        else:
            code.putln(
                "PyErr_SetNone(PyExc_AssertionError);")
        code.putln(
                code.error_goto(self.pos))
        code.putln(
            "}")
        self.cond.generate_disposal_code(code)
5681
        self.cond.free_temps(code)
5682 5683
        code.putln(
            "}")
5684
        code.putln("#endif")
William Stein's avatar
William Stein committed
5685

5686 5687 5688 5689 5690
    def generate_function_definitions(self, env, code):
        self.cond.generate_function_definitions(env, code)
        if self.value is not None:
            self.value.generate_function_definitions(env, code)

5691 5692 5693 5694 5695 5696
    def annotate(self, code):
        self.cond.annotate(code)
        if self.value:
            self.value.annotate(code)


William Stein's avatar
William Stein committed
5697 5698 5699 5700 5701
class IfStatNode(StatNode):
    #  if statement
    #
    #  if_clauses   [IfClauseNode]
    #  else_clause  StatNode or None
5702 5703

    child_attrs = ["if_clauses", "else_clause"]
5704

William Stein's avatar
William Stein committed
5705 5706 5707 5708 5709
    def analyse_declarations(self, env):
        for if_clause in self.if_clauses:
            if_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
5710

William Stein's avatar
William Stein committed
5711
    def analyse_expressions(self, env):
5712 5713
        self.if_clauses = [ if_clause.analyse_expressions(env)
                            for if_clause in self.if_clauses ]
William Stein's avatar
William Stein committed
5714
        if self.else_clause:
5715 5716
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
5717

William Stein's avatar
William Stein committed
5718
    def generate_execution_code(self, code):
5719
        code.mark_pos(self.pos)
5720 5721 5722 5723 5724
        end_label = code.new_label()
        for if_clause in self.if_clauses:
            if_clause.generate_execution_code(code, end_label)
        if self.else_clause:
            code.putln("/*else*/ {")
William Stein's avatar
William Stein committed
5725
            self.else_clause.generate_execution_code(code)
5726 5727
            code.putln("}")
        code.put_label(end_label)
5728

5729 5730 5731 5732 5733
    def generate_function_definitions(self, env, code):
        for clause in self.if_clauses:
            clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
5734

5735 5736 5737 5738 5739
    def annotate(self, code):
        for if_clause in self.if_clauses:
            if_clause.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
5740 5741 5742 5743 5744 5745 5746


class IfClauseNode(Node):
    #  if or elif clause in an if statement
    #
    #  condition   ExprNode
    #  body        StatNode
5747

5748 5749
    child_attrs = ["condition", "body"]

William Stein's avatar
William Stein committed
5750 5751
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
5752

William Stein's avatar
William Stein committed
5753 5754 5755
    def analyse_expressions(self, env):
        self.condition = \
            self.condition.analyse_temp_boolean_expression(env)
5756 5757
        self.body = self.body.analyse_expressions(env)
        return self
5758

William Stein's avatar
William Stein committed
5759 5760 5761 5762
    def generate_execution_code(self, code, end_label):
        self.condition.generate_evaluation_code(code)
        code.putln(
            "if (%s) {" %
5763
                self.condition.result())
5764 5765
        self.condition.generate_disposal_code(code)
        self.condition.free_temps(code)
William Stein's avatar
William Stein committed
5766
        self.body.generate_execution_code(code)
5767 5768
        if not self.body.is_terminator:
            code.put_goto(end_label)
William Stein's avatar
William Stein committed
5769
        code.putln("}")
5770

5771 5772 5773 5774
    def generate_function_definitions(self, env, code):
        self.condition.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)

5775 5776 5777
    def annotate(self, code):
        self.condition.annotate(code)
        self.body.annotate(code)
5778

5779 5780 5781 5782 5783 5784

class SwitchCaseNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # conditions    [ExprNode]
    # body          StatNode
5785

5786
    child_attrs = ['conditions', 'body']
5787

5788 5789
    def generate_execution_code(self, code):
        for cond in self.conditions:
5790
            code.mark_pos(cond.pos)
5791 5792
            cond.generate_evaluation_code(code)
            code.putln("case %s:" % cond.result())
5793 5794
        self.body.generate_execution_code(code)
        code.putln("break;")
5795 5796 5797 5798 5799

    def generate_function_definitions(self, env, code):
        for cond in self.conditions:
            cond.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
5800

5801 5802 5803
    def annotate(self, code):
        for cond in self.conditions:
            cond.annotate(code)
5804
        self.body.annotate(code)
5805 5806 5807 5808 5809 5810 5811

class SwitchStatNode(StatNode):
    # Generated in the optimization of an if-elif-else node
    #
    # test          ExprNode
    # cases         [SwitchCaseNode]
    # else_clause   StatNode or None
5812

5813
    child_attrs = ['test', 'cases', 'else_clause']
5814

5815
    def generate_execution_code(self, code):
5816
        self.test.generate_evaluation_code(code)
5817
        code.putln("switch (%s) {" % self.test.result())
5818 5819 5820 5821 5822
        for case in self.cases:
            case.generate_execution_code(code)
        if self.else_clause is not None:
            code.putln("default:")
            self.else_clause.generate_execution_code(code)
5823
            code.putln("break;")
5824 5825 5826 5827 5828
        else:
            # Always generate a default clause to prevent C compiler warnings
            # about unmatched enum values (it was not the user who decided to
            # generate the switch statement, so shouldn't be bothered).
            code.putln("default: break;")
5829 5830
        code.putln("}")

5831 5832 5833 5834 5835 5836 5837
    def generate_function_definitions(self, env, code):
        self.test.generate_function_definitions(env, code)
        for case in self.cases:
            case.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

5838 5839 5840 5841
    def annotate(self, code):
        self.test.annotate(code)
        for case in self.cases:
            case.annotate(code)
5842 5843
        if self.else_clause is not None:
            self.else_clause.annotate(code)
5844

5845
class LoopNode(object):
5846
    pass
5847

5848

5849
class WhileStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
5850 5851 5852 5853 5854
    #  while statement
    #
    #  condition    ExprNode
    #  body         StatNode
    #  else_clause  StatNode
5855 5856

    child_attrs = ["condition", "body", "else_clause"]
5857

William Stein's avatar
William Stein committed
5858 5859 5860 5861
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
5862

William Stein's avatar
William Stein committed
5863
    def analyse_expressions(self, env):
5864 5865
        if self.condition:
            self.condition = self.condition.analyse_temp_boolean_expression(env)
5866
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
5867
        if self.else_clause:
5868 5869
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
5870

William Stein's avatar
William Stein committed
5871 5872 5873 5874
    def generate_execution_code(self, code):
        old_loop_labels = code.new_loop_labels()
        code.putln(
            "while (1) {")
5875 5876 5877 5878 5879 5880 5881
        if self.condition:
            self.condition.generate_evaluation_code(code)
            self.condition.generate_disposal_code(code)
            code.putln(
                "if (!%s) break;" %
                    self.condition.result())
            self.condition.free_temps(code)
William Stein's avatar
William Stein committed
5882
        self.body.generate_execution_code(code)
5883
        code.put_label(code.continue_label)
William Stein's avatar
William Stein committed
5884 5885 5886 5887
        code.putln("}")
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
5888
            code.mark_pos(self.else_clause.pos)
William Stein's avatar
William Stein committed
5889 5890 5891 5892 5893
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)

5894
    def generate_function_definitions(self, env, code):
5895 5896
        if self.condition:
            self.condition.generate_function_definitions(env, code)
5897 5898 5899 5900
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

5901
    def annotate(self, code):
5902 5903
        if self.condition:
            self.condition.annotate(code)
5904 5905 5906 5907
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
5908

5909 5910 5911 5912
class DictIterationNextNode(Node):
    # Helper node for calling PyDict_Next() inside of a WhileStatNode
    # and checking the dictionary size for changes.  Created in
    # Optimize.py.
5913 5914 5915
    child_attrs = ['dict_obj', 'expected_size', 'pos_index_var',
                   'coerced_key_var', 'coerced_value_var', 'coerced_tuple_var',
                   'key_target', 'value_target', 'tuple_target', 'is_dict_flag']
5916

5917 5918 5919 5920 5921 5922
    coerced_key_var = key_ref = None
    coerced_value_var = value_ref = None
    coerced_tuple_var = tuple_ref = None

    def __init__(self, dict_obj, expected_size, pos_index_var,
                 key_target, value_target, tuple_target, is_dict_flag):
5923 5924 5925 5926
        Node.__init__(
            self, dict_obj.pos,
            dict_obj = dict_obj,
            expected_size = expected_size,
5927 5928 5929 5930 5931 5932
            pos_index_var = pos_index_var,
            key_target = key_target,
            value_target = value_target,
            tuple_target = tuple_target,
            is_dict_flag = is_dict_flag,
            is_temp = True,
5933 5934 5935
            type = PyrexTypes.c_bint_type)

    def analyse_expressions(self, env):
5936
        from . import ExprNodes
5937 5938 5939 5940
        self.dict_obj = self.dict_obj.analyse_types(env)
        self.expected_size = self.expected_size.analyse_types(env)
        if self.pos_index_var:
            self.pos_index_var = self.pos_index_var.analyse_types(env)
5941
        if self.key_target:
5942
            self.key_target = self.key_target.analyse_target_types(env)
5943
            self.key_ref = ExprNodes.TempNode(self.key_target.pos, PyrexTypes.py_object_type)
5944 5945
            self.coerced_key_var = self.key_ref.coerce_to(self.key_target.type, env)
        if self.value_target:
5946
            self.value_target = self.value_target.analyse_target_types(env)
5947
            self.value_ref = ExprNodes.TempNode(self.value_target.pos, type=PyrexTypes.py_object_type)
5948 5949
            self.coerced_value_var = self.value_ref.coerce_to(self.value_target.type, env)
        if self.tuple_target:
5950
            self.tuple_target = self.tuple_target.analyse_target_types(env)
5951
            self.tuple_ref = ExprNodes.TempNode(self.tuple_target.pos, PyrexTypes.py_object_type)
5952
            self.coerced_tuple_var = self.tuple_ref.coerce_to(self.tuple_target.type, env)
5953 5954
        self.is_dict_flag = self.is_dict_flag.analyse_types(env)
        return self
5955 5956 5957 5958 5959

    def generate_function_definitions(self, env, code):
        self.dict_obj.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
5960
        code.globalstate.use_utility_code(UtilityCode.load_cached("dict_iter", "Optimize.c"))
5961 5962
        self.dict_obj.generate_evaluation_code(code)

5963 5964 5965 5966 5967 5968 5969 5970
        assignments = []
        temp_addresses = []
        for var, result, target in [(self.key_ref, self.coerced_key_var, self.key_target),
                                    (self.value_ref, self.coerced_value_var, self.value_target),
                                    (self.tuple_ref, self.coerced_tuple_var, self.tuple_target)]:
            if target is None:
                addr = 'NULL'
            else:
5971 5972 5973
                assignments.append((var, result, target))
                var.allocate(code)
                addr = '&%s' % var.result()
5974 5975 5976 5977 5978
            temp_addresses.append(addr)

        result_temp = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
        code.putln("%s = __Pyx_dict_iter_next(%s, %s, &%s, %s, %s, %s, %s);" % (
            result_temp,
5979
            self.dict_obj.py_result(),
5980 5981 5982 5983 5984 5985 5986 5987 5988 5989 5990 5991
            self.expected_size.result(),
            self.pos_index_var.result(),
            temp_addresses[0],
            temp_addresses[1],
            temp_addresses[2],
            self.is_dict_flag.result()
        ))
        code.putln("if (unlikely(%s == 0)) break;" % result_temp)
        code.putln(code.error_goto_if("%s == -1" % result_temp, self.pos))
        code.funcstate.release_temp(result_temp)

        # evaluate all coercions before the assignments
5992 5993
        for var, result, target in assignments:
            code.put_gotref(var.result())
5994
        for var, result, target in assignments:
5995
            result.generate_evaluation_code(code)
5996
        for var, result, target in assignments:
5997
            target.generate_assignment_code(result, code)
5998
            var.release(code)
5999

Robert Bradshaw's avatar
Robert Bradshaw committed
6000
def ForStatNode(pos, **kw):
6001
    if 'iterator' in kw:
Robert Bradshaw's avatar
Robert Bradshaw committed
6002 6003 6004 6005
        return ForInStatNode(pos, **kw)
    else:
        return ForFromStatNode(pos, **kw)

6006
class ForInStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6007 6008 6009 6010 6011 6012 6013
    #  for statement
    #
    #  target        ExprNode
    #  iterator      IteratorNode
    #  body          StatNode
    #  else_clause   StatNode
    #  item          NextNode       used internally
6014

6015
    child_attrs = ["target", "iterator", "body", "else_clause"]
6016
    item = None
6017

William Stein's avatar
William Stein committed
6018
    def analyse_declarations(self, env):
6019
        from . import ExprNodes
William Stein's avatar
William Stein committed
6020 6021 6022 6023
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6024
        self.item = ExprNodes.NextNode(self.iterator)
6025

William Stein's avatar
William Stein committed
6026
    def analyse_expressions(self, env):
6027
        self.target = self.target.analyse_target_types(env)
6028
        self.iterator = self.iterator.analyse_expressions(env)
6029
        from . import ExprNodes
6030
        self.item = ExprNodes.NextNode(self.iterator)  # must rewrap after analysis
6031
        self.item = self.item.analyse_expressions(env)
6032 6033
        if (self.iterator.type.is_ptr or self.iterator.type.is_array) and \
            self.target.type.assignable_from(self.iterator.type):
6034 6035 6036 6037
            # C array slice optimization.
            pass
        else:
            self.item = self.item.coerce_to(self.target.type, env)
6038
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6039
        if self.else_clause:
6040 6041
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
William Stein's avatar
William Stein committed
6042 6043 6044 6045

    def generate_execution_code(self, code):
        old_loop_labels = code.new_loop_labels()
        self.iterator.generate_evaluation_code(code)
Mark Florisson's avatar
Mark Florisson committed
6046
        code.putln("for (;;) {")
William Stein's avatar
William Stein committed
6047 6048 6049
        self.item.generate_evaluation_code(code)
        self.target.generate_assignment_code(self.item, code)
        self.body.generate_execution_code(code)
6050
        code.mark_pos(self.pos)
6051
        code.put_label(code.continue_label)
Mark Florisson's avatar
Mark Florisson committed
6052
        code.putln("}")
William Stein's avatar
William Stein committed
6053 6054
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
6055

William Stein's avatar
William Stein committed
6056
        if self.else_clause:
6057 6058 6059 6060 6061 6062 6063
            # in nested loops, the 'else' block can contain a
            # 'continue' statement for the outer loop, but we may need
            # to generate cleanup code before taking that path, so we
            # intercept it here
            orig_continue_label = code.continue_label
            code.continue_label = code.new_label('outer_continue')

William Stein's avatar
William Stein committed
6064 6065 6066
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
6067 6068 6069

            if code.label_used(code.continue_label):
                code.put_goto(break_label)
6070
                code.mark_pos(self.pos)
6071 6072 6073 6074 6075
                code.put_label(code.continue_label)
                self.iterator.generate_disposal_code(code)
                code.put_goto(orig_continue_label)
            code.set_loop_labels(old_loop_labels)

6076
        code.mark_pos(self.pos)
6077 6078
        if code.label_used(break_label):
            code.put_label(break_label)
William Stein's avatar
William Stein committed
6079
        self.iterator.generate_disposal_code(code)
6080
        self.iterator.free_temps(code)
William Stein's avatar
William Stein committed
6081

6082 6083 6084 6085 6086 6087 6088
    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.iterator.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6089 6090 6091 6092 6093 6094 6095 6096
    def annotate(self, code):
        self.target.annotate(code)
        self.iterator.annotate(code)
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
        self.item.annotate(code)

William Stein's avatar
William Stein committed
6097

6098
class ForFromStatNode(LoopNode, StatNode):
William Stein's avatar
William Stein committed
6099 6100 6101 6102 6103 6104 6105
    #  for name from expr rel name rel expr
    #
    #  target        NameNode
    #  bound1        ExprNode
    #  relation1     string
    #  relation2     string
    #  bound2        ExprNode
6106
    #  step          ExprNode or None
William Stein's avatar
William Stein committed
6107 6108 6109 6110 6111
    #  body          StatNode
    #  else_clause   StatNode or None
    #
    #  Used internally:
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
6112
    #  from_range         bool
6113
    #  is_py_target       bool
6114
    #  loopvar_node       ExprNode (usually a NameNode or temp node)
William Stein's avatar
William Stein committed
6115
    #  py_loopvar_node    PyTempNode or None
6116
    child_attrs = ["target", "bound1", "bound2", "step", "body", "else_clause"]
6117 6118

    is_py_target = False
6119
    loopvar_node = None
6120
    py_loopvar_node = None
Robert Bradshaw's avatar
Robert Bradshaw committed
6121
    from_range = False
6122

6123 6124 6125 6126 6127 6128 6129
    gil_message = "For-loop using object bounds or target"

    def nogil_check(self, env):
        for x in (self.target, self.bound1, self.bound2):
            if x.type.is_pyobject:
                self.gil_error()

Robert Bradshaw's avatar
Robert Bradshaw committed
6130 6131 6132 6133 6134
    def analyse_declarations(self, env):
        self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6135

William Stein's avatar
William Stein committed
6136
    def analyse_expressions(self, env):
6137
        from . import ExprNodes
6138 6139 6140
        self.target = self.target.analyse_target_types(env)
        self.bound1 = self.bound1.analyse_types(env)
        self.bound2 = self.bound2.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6141 6142
        if self.step is not None:
            if isinstance(self.step, ExprNodes.UnaryMinusNode):
Stefan Behnel's avatar
Stefan Behnel committed
6143
                warning(self.step.pos, "Probable infinite loop in for-from-by statement. Consider switching the directions of the relations.", 2)
6144
            self.step = self.step.analyse_types(env)
6145

6146
        if self.target.type.is_numeric:
Robert Bradshaw's avatar
Robert Bradshaw committed
6147
            loop_type = self.target.type
6148
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6149 6150 6151 6152 6153 6154 6155 6156 6157
            loop_type = PyrexTypes.c_int_type
            if not self.bound1.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound1.type)
            if not self.bound2.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.bound2.type)
            if self.step is not None and not self.step.type.is_pyobject:
                loop_type = PyrexTypes.widest_numeric_type(loop_type, self.step.type)
        self.bound1 = self.bound1.coerce_to(loop_type, env)
        self.bound2 = self.bound2.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6158 6159
        if not self.bound2.is_literal:
            self.bound2 = self.bound2.coerce_to_temp(env)
6160
        if self.step is not None:
6161
            self.step = self.step.coerce_to(loop_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6162 6163
            if not self.step.is_literal:
                self.step = self.step.coerce_to_temp(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
6164

William Stein's avatar
William Stein committed
6165
        target_type = self.target.type
6166
        if not (target_type.is_pyobject or target_type.is_numeric):
6167
            error(self.target.pos,
Robert Bradshaw's avatar
Robert Bradshaw committed
6168
                "for-from loop variable must be c numeric type or Python object")
6169
        if target_type.is_numeric:
Robert Bradshaw's avatar
Robert Bradshaw committed
6170
            self.is_py_target = False
6171 6172
            if isinstance(self.target, ExprNodes.IndexNode) and self.target.is_buffer_access:
                raise error(self.pos, "Buffer indexing not allowed as for loop target.")
6173
            self.loopvar_node = self.target
William Stein's avatar
William Stein committed
6174 6175
            self.py_loopvar_node = None
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6176 6177
            self.is_py_target = True
            c_loopvar_node = ExprNodes.TempNode(self.pos, loop_type, env)
6178
            self.loopvar_node = c_loopvar_node
William Stein's avatar
William Stein committed
6179 6180
            self.py_loopvar_node = \
                ExprNodes.CloneNode(c_loopvar_node).coerce_to_pyobject(env)
6181
        self.body = self.body.analyse_expressions(env)
William Stein's avatar
William Stein committed
6182
        if self.else_clause:
6183 6184
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6185

William Stein's avatar
William Stein committed
6186 6187
    def generate_execution_code(self, code):
        old_loop_labels = code.new_loop_labels()
Robert Bradshaw's avatar
Robert Bradshaw committed
6188
        from_range = self.from_range
William Stein's avatar
William Stein committed
6189 6190 6191
        self.bound1.generate_evaluation_code(code)
        self.bound2.generate_evaluation_code(code)
        offset, incop = self.relation_table[self.relation1]
6192 6193
        if self.step is not None:
            self.step.generate_evaluation_code(code)
Magnus Lie Hetland's avatar
Magnus Lie Hetland committed
6194 6195
            step = self.step.result()
            incop = "%s=%s" % (incop[0], step)
6196
        from . import ExprNodes
6197 6198 6199 6200
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.allocate(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.allocate(code)
6201
        if from_range:
Robert Bradshaw's avatar
Robert Bradshaw committed
6202
            loopvar_name = code.funcstate.allocate_temp(self.target.type, False)
6203
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
6204
            loopvar_name = self.loopvar_node.result()
6205 6206 6207 6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222
        if self.target.type.is_int and not self.target.type.signed and self.relation2[0] == '>':
            # Handle the case where the endpoint of an unsigned int iteration
            # is within step of 0.
            if not self.step:
                step = 1
            code.putln(
                "for (%s = %s%s + %s; %s %s %s + %s; ) { %s%s;" % (
                    loopvar_name,
                    self.bound1.result(), offset, step,
                    loopvar_name, self.relation2, self.bound2.result(), step,
                    loopvar_name, incop))
        else:
            code.putln(
                "for (%s = %s%s; %s %s %s; %s%s) {" % (
                    loopvar_name,
                    self.bound1.result(), offset,
                    loopvar_name, self.relation2, self.bound2.result(),
                    loopvar_name, incop))
William Stein's avatar
William Stein committed
6223 6224 6225
        if self.py_loopvar_node:
            self.py_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(self.py_loopvar_node, code)
Robert Bradshaw's avatar
Robert Bradshaw committed
6226 6227 6228
        elif from_range:
            code.putln("%s = %s;" % (
                            self.target.result(), loopvar_name))
William Stein's avatar
William Stein committed
6229 6230
        self.body.generate_execution_code(code)
        code.put_label(code.continue_label)
Robert Bradshaw's avatar
Robert Bradshaw committed
6231
        if self.py_loopvar_node:
6232 6233 6234
            # This mess is to make for..from loops with python targets behave
            # exactly like those with C targets with regards to re-assignment
            # of the loop variable.
6235
            if self.target.entry.is_pyglobal:
6236
                # We know target is a NameNode, this is the only ugly case.
6237
                target_node = ExprNodes.PyTempNode(self.target.pos, None)
6238 6239
                target_node.allocate(code)
                interned_cname = code.intern_identifier(self.target.entry.name)
6240 6241 6242 6243 6244 6245 6246 6247 6248 6249
                if self.target.entry.scope.is_module_scope:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetModuleGlobalName", "ObjectHandling.c"))
                    lookup_func = '__Pyx_GetModuleGlobalName(%s)'
                else:
                    code.globalstate.use_utility_code(
                        UtilityCode.load_cached("GetNameInClass", "ObjectHandling.c"))
                    lookup_func = '__Pyx_GetNameInClass(%s, %%s)' % (
                        self.target.entry.scope.namespace_cname)
                code.putln("%s = %s; %s" % (
6250
                    target_node.result(),
6251
                    lookup_func % interned_cname,
6252
                    code.error_goto_if_null(target_node.result(), self.target.pos)))
6253
                code.put_gotref(target_node.result())
6254 6255
            else:
                target_node = self.target
6256 6257
            from_py_node = ExprNodes.CoerceFromPyTypeNode(
                self.loopvar_node.type, target_node, self.target.entry.scope)
6258 6259
            from_py_node.temp_code = loopvar_name
            from_py_node.generate_result_code(code)
6260
            if self.target.entry.is_pyglobal:
6261 6262
                code.put_decref(target_node.result(), target_node.type)
                target_node.release(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
6263
        code.putln("}")
6264
        if self.py_loopvar_node:
6265 6266
            # This is potentially wasteful, but we don't want the semantics to
            # depend on whether or not the loop is a python type.
6267 6268
            self.py_loopvar_node.generate_evaluation_code(code)
            self.target.generate_assignment_code(self.py_loopvar_node, code)
6269 6270
        if from_range:
            code.funcstate.release_temp(loopvar_name)
William Stein's avatar
William Stein committed
6271 6272 6273 6274 6275 6276 6277 6278
        break_label = code.break_label
        code.set_loop_labels(old_loop_labels)
        if self.else_clause:
            code.putln("/*else*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln("}")
        code.put_label(break_label)
        self.bound1.generate_disposal_code(code)
6279
        self.bound1.free_temps(code)
William Stein's avatar
William Stein committed
6280
        self.bound2.generate_disposal_code(code)
6281
        self.bound2.free_temps(code)
6282 6283 6284 6285
        if isinstance(self.loopvar_node, ExprNodes.TempNode):
            self.loopvar_node.release(code)
        if isinstance(self.py_loopvar_node, ExprNodes.TempNode):
            self.py_loopvar_node.release(code)
6286 6287
        if self.step is not None:
            self.step.generate_disposal_code(code)
6288
            self.step.free_temps(code)
6289

William Stein's avatar
William Stein committed
6290 6291 6292 6293 6294 6295 6296
    relation_table = {
        # {relop : (initial offset, increment op)}
        '<=': ("",   "++"),
        '<' : ("+1", "++"),
        '>=': ("",   "--"),
        '>' : ("-1", "--")
    }
6297 6298 6299 6300 6301 6302 6303 6304 6305 6306

    def generate_function_definitions(self, env, code):
        self.target.generate_function_definitions(env, code)
        self.bound1.generate_function_definitions(env, code)
        self.bound2.generate_function_definitions(env, code)
        if self.step is not None:
            self.step.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)
6307

6308 6309 6310 6311 6312
    def annotate(self, code):
        self.target.annotate(code)
        self.bound1.annotate(code)
        self.bound2.annotate(code)
        if self.step:
6313
            self.step.annotate(code)
6314 6315 6316
        self.body.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)
William Stein's avatar
William Stein committed
6317 6318


6319 6320 6321
class WithStatNode(StatNode):
    """
    Represents a Python with statement.
6322

6323
    Implemented by the WithTransform as follows:
6324 6325 6326 6327 6328 6329 6330 6331 6332 6333 6334 6335 6336 6337 6338 6339 6340

        MGR = EXPR
        EXIT = MGR.__exit__
        VALUE = MGR.__enter__()
        EXC = True
        try:
            try:
                TARGET = VALUE  # optional
                BODY
            except:
                EXC = False
                if not EXIT(*EXCINFO):
                    raise
        finally:
            if EXC:
                EXIT(None, None, None)
            MGR = EXIT = VALUE = None
6341 6342
    """
    #  manager          The with statement manager object
6343
    #  target           ExprNode  the target lhs of the __enter__() call
6344
    #  body             StatNode
6345
    #  enter_call       ExprNode  the call to the __enter__() method
6346
    #  exit_var         String    the cname of the __exit__() method reference
6347

6348
    child_attrs = ["manager", "enter_call", "target", "body"]
6349

6350
    enter_call = None
6351
    target_temp = None
6352 6353 6354

    def analyse_declarations(self, env):
        self.manager.analyse_declarations(env)
6355
        self.enter_call.analyse_declarations(env)
6356 6357 6358
        self.body.analyse_declarations(env)

    def analyse_expressions(self, env):
6359 6360
        self.manager = self.manager.analyse_types(env)
        self.enter_call = self.enter_call.analyse_types(env)
6361 6362 6363 6364
        if self.target:
            # set up target_temp before descending into body (which uses it)
            from .ExprNodes import TempNode
            self.target_temp = TempNode(self.enter_call.pos, self.enter_call.type)
6365 6366
        self.body = self.body.analyse_expressions(env)
        return self
6367

6368 6369
    def generate_function_definitions(self, env, code):
        self.manager.generate_function_definitions(env, code)
6370
        self.enter_call.generate_function_definitions(env, code)
6371 6372
        self.body.generate_function_definitions(env, code)

6373 6374 6375 6376
    def generate_execution_code(self, code):
        code.putln("/*with:*/ {")
        self.manager.generate_evaluation_code(code)
        self.exit_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
6377
        code.globalstate.use_utility_code(
6378 6379
            UtilityCode.load_cached("PyObjectLookupSpecial", "ObjectHandling.c"))
        code.putln("%s = __Pyx_PyObject_LookupSpecial(%s, %s); %s" % (
6380 6381
            self.exit_var,
            self.manager.py_result(),
6382
            code.intern_identifier(EncodedString('__exit__')),
6383 6384 6385 6386 6387 6388 6389 6390
            code.error_goto_if_null(self.exit_var, self.pos),
            ))
        code.put_gotref(self.exit_var)

        # need to free exit_var in the face of exceptions during setup
        old_error_label = code.new_error_label()
        intermediate_error_label = code.error_label

6391
        self.enter_call.generate_evaluation_code(code)
6392 6393 6394 6395 6396 6397 6398
        if self.target:
            # The temp result will be cleaned up by the WithTargetAssignmentStatNode
            # after assigning its result to the target of the 'with' statement.
            self.target_temp.allocate(code)
            self.enter_call.make_owned_reference(code)
            code.putln("%s = %s;" % (self.target_temp.result(), self.enter_call.result()))
            self.enter_call.generate_post_assignment_code(code)
6399
        else:
6400 6401 6402
            self.enter_call.generate_disposal_code(code)
        self.enter_call.free_temps(code)

6403 6404 6405 6406 6407 6408
        self.manager.generate_disposal_code(code)
        self.manager.free_temps(code)

        code.error_label = old_error_label
        self.body.generate_execution_code(code)

6409 6410 6411 6412 6413 6414 6415
        if code.label_used(intermediate_error_label):
            step_over_label = code.new_label()
            code.put_goto(step_over_label)
            code.put_label(intermediate_error_label)
            code.put_decref_clear(self.exit_var, py_object_type)
            code.put_goto(old_error_label)
            code.put_label(step_over_label)
6416 6417 6418 6419

        code.funcstate.release_temp(self.exit_var)
        code.putln('}')

6420

6421 6422 6423 6424
class WithTargetAssignmentStatNode(AssignmentNode):
    # The target assignment of the 'with' statement value (return
    # value of the __enter__() call).
    #
6425
    # This is a special cased assignment that properly cleans up the RHS.
6426
    #
6427 6428 6429
    # lhs       ExprNode      the assignment target
    # rhs       ExprNode      a (coerced) TempNode for the rhs (from WithStatNode)
    # with_node WithStatNode  the surrounding with-statement
6430

6431 6432 6433
    child_attrs = ["rhs", "lhs"]
    with_node = None
    rhs = None
6434 6435 6436 6437

    def analyse_declarations(self, env):
        self.lhs.analyse_target_declaration(env)

6438
    def analyse_expressions(self, env):
6439
        self.lhs = self.lhs.analyse_target_types(env)
6440
        self.lhs.gil_assignment_check(env)
6441
        self.rhs = self.with_node.target_temp.coerce_to(self.lhs.type, env)
6442
        return self
6443 6444 6445 6446

    def generate_execution_code(self, code):
        self.rhs.generate_evaluation_code(code)
        self.lhs.generate_assignment_code(self.rhs, code)
6447
        self.with_node.target_temp.release(code)
6448 6449 6450 6451 6452

    def annotate(self, code):
        self.lhs.annotate(code)
        self.rhs.annotate(code)

6453

William Stein's avatar
William Stein committed
6454 6455 6456 6457 6458 6459
class TryExceptStatNode(StatNode):
    #  try .. except statement
    #
    #  body             StatNode
    #  except_clauses   [ExceptClauseNode]
    #  else_clause      StatNode or None
6460

6461
    child_attrs = ["body", "except_clauses", "else_clause"]
6462

William Stein's avatar
William Stein committed
6463 6464 6465 6466 6467 6468
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        for except_clause in self.except_clauses:
            except_clause.analyse_declarations(env)
        if self.else_clause:
            self.else_clause.analyse_declarations(env)
6469

William Stein's avatar
William Stein committed
6470
    def analyse_expressions(self, env):
6471
        self.body = self.body.analyse_expressions(env)
6472
        default_clause_seen = 0
6473 6474
        for i, except_clause in enumerate(self.except_clauses):
            except_clause = self.except_clauses[i] = except_clause.analyse_expressions(env)
6475 6476 6477 6478 6479
            if default_clause_seen:
                error(except_clause.pos, "default 'except:' must be last")
            if not except_clause.pattern:
                default_clause_seen = 1
        self.has_default_clause = default_clause_seen
William Stein's avatar
William Stein committed
6480
        if self.else_clause:
6481 6482
            self.else_clause = self.else_clause.analyse_expressions(env)
        return self
6483

6484
    nogil_check = Node.gil_error
6485 6486
    gil_message = "Try-except statement"

William Stein's avatar
William Stein committed
6487
    def generate_execution_code(self, code):
6488
        old_return_label = code.return_label
6489
        old_break_label = code.break_label
6490
        old_continue_label = code.continue_label
William Stein's avatar
William Stein committed
6491 6492
        old_error_label = code.new_error_label()
        our_error_label = code.error_label
6493 6494 6495
        except_end_label = code.new_label('exception_handled')
        except_error_label = code.new_label('except_error')
        except_return_label = code.new_label('except_return')
6496
        try_return_label = code.new_label('try_return')
6497
        try_break_label = code.new_label('try_break')
6498
        try_continue_label = code.new_label('try_continue')
6499
        try_end_label = code.new_label('try_end')
6500

6501
        exc_save_vars = [code.funcstate.allocate_temp(py_object_type, False)
Stefan Behnel's avatar
Stefan Behnel committed
6502
                         for _ in xrange(3)]
6503
        code.putln("{")
6504
        save_exc = code.insertion_point()
William Stein's avatar
William Stein committed
6505 6506
        code.putln(
            "/*try:*/ {")
6507
        code.return_label = try_return_label
6508
        code.break_label = try_break_label
6509
        code.continue_label = try_continue_label
William Stein's avatar
William Stein committed
6510 6511 6512
        self.body.generate_execution_code(code)
        code.putln(
            "}")
6513
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
6514 6515 6516 6517
        can_raise = code.label_used(our_error_label)

        if can_raise:
            # inject code before the try block to save away the exception state
6518
            code.globalstate.use_utility_code(reset_exception_utility_code)
6519 6520 6521 6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534 6535 6536
            save_exc.putln("__Pyx_ExceptionSave(%s);" %
                           ', '.join(['&%s' % var for var in exc_save_vars]))
            for var in exc_save_vars:
                save_exc.put_xgotref(var)

            def restore_saved_exception():
                for name in exc_save_vars:
                    code.put_xgiveref(name)
                code.putln("__Pyx_ExceptionReset(%s);" %
                           ', '.join(exc_save_vars))
        else:
            # try block cannot raise exceptions, but we had to allocate the temps above,
            # so just keep the C compiler from complaining about them being unused
            save_exc.putln("if (%s); else {/*mark used*/};" % '||'.join(exc_save_vars))

            def restore_saved_exception():
                pass

6537 6538
        code.error_label = except_error_label
        code.return_label = except_return_label
6539
        normal_case_terminates = self.body.is_terminator
William Stein's avatar
William Stein committed
6540 6541 6542 6543 6544 6545
        if self.else_clause:
            code.putln(
                "/*else:*/ {")
            self.else_clause.generate_execution_code(code)
            code.putln(
                "}")
6546 6547
            if not normal_case_terminates:
                normal_case_terminates = self.else_clause.is_terminator
6548

6549
        if can_raise:
6550 6551 6552 6553
            if not normal_case_terminates:
                for var in exc_save_vars:
                    code.put_xdecref_clear(var, py_object_type)
                code.put_goto(try_end_label)
6554 6555 6556 6557 6558 6559 6560 6561 6562 6563 6564 6565 6566
            code.put_label(our_error_label)
            for temp_name, temp_type in temps_to_clean_up:
                code.put_xdecref_clear(temp_name, temp_type)
            for except_clause in self.except_clauses:
                except_clause.generate_handling_code(code, except_end_label)
            if not self.has_default_clause:
                code.put_goto(except_error_label)

        for exit_label, old_label in [(except_error_label, old_error_label),
                                      (try_break_label, old_break_label),
                                      (try_continue_label, old_continue_label),
                                      (try_return_label, old_return_label),
                                      (except_return_label, old_return_label)]:
6567
            if code.label_used(exit_label):
6568
                if not normal_case_terminates and not code.label_used(try_end_label):
6569
                    code.put_goto(try_end_label)
6570
                code.put_label(exit_label)
6571
                restore_saved_exception()
6572
                code.put_goto(old_label)
6573 6574

        if code.label_used(except_end_label):
6575
            if not normal_case_terminates and not code.label_used(try_end_label):
6576
                code.put_goto(try_end_label)
6577
            code.put_label(except_end_label)
6578 6579 6580
            restore_saved_exception()
        if code.label_used(try_end_label):
            code.put_label(try_end_label)
6581 6582
        code.putln("}")

6583 6584 6585
        for cname in exc_save_vars:
            code.funcstate.release_temp(cname)

6586
        code.return_label = old_return_label
6587
        code.break_label = old_break_label
6588
        code.continue_label = old_continue_label
6589
        code.error_label = old_error_label
William Stein's avatar
William Stein committed
6590

6591 6592 6593 6594 6595 6596 6597
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        for except_clause in self.except_clauses:
            except_clause.generate_function_definitions(env, code)
        if self.else_clause is not None:
            self.else_clause.generate_function_definitions(env, code)

6598 6599 6600 6601 6602 6603 6604
    def annotate(self, code):
        self.body.annotate(code)
        for except_node in self.except_clauses:
            except_node.annotate(code)
        if self.else_clause:
            self.else_clause.annotate(code)

William Stein's avatar
William Stein committed
6605 6606 6607 6608

class ExceptClauseNode(Node):
    #  Part of try ... except statement.
    #
6609
    #  pattern        [ExprNode]
William Stein's avatar
William Stein committed
6610 6611
    #  target         ExprNode or None
    #  body           StatNode
6612
    #  excinfo_target TupleNode(3*ResultRefNode) or None   optional target for exception info (not owned here!)
William Stein's avatar
William Stein committed
6613 6614 6615
    #  match_flag     string             result of exception match
    #  exc_value      ExcValueNode       used internally
    #  function_name  string             qualified name of enclosing function
6616
    #  exc_vars       (string * 3)       local exception variables
6617
    #  is_except_as   bool               Py3-style "except ... as xyz"
6618 6619 6620 6621

    # excinfo_target is never set by the parser, but can be set by a transform
    # in order to extract more extensive information about the exception as a
    # sys.exc_info()-style tuple into a target variable
6622

6623
    child_attrs = ["pattern", "target", "body", "exc_value"]
6624

6625
    exc_value = None
6626
    excinfo_target = None
6627
    is_except_as = False
6628

William Stein's avatar
William Stein committed
6629 6630 6631 6632
    def analyse_declarations(self, env):
        if self.target:
            self.target.analyse_target_declaration(env)
        self.body.analyse_declarations(env)
6633

William Stein's avatar
William Stein committed
6634 6635 6636
    def analyse_expressions(self, env):
        self.function_name = env.qualified_name
        if self.pattern:
6637 6638
            # normalise/unpack self.pattern into a list
            for i, pattern in enumerate(self.pattern):
6639
                pattern = pattern.analyse_expressions(env)
6640
                self.pattern[i] = pattern.coerce_to_pyobject(env)
6641

William Stein's avatar
William Stein committed
6642
        if self.target:
6643
            from . import ExprNodes
6644
            self.exc_value = ExprNodes.ExcValueNode(self.pos)
6645
            self.target = self.target.analyse_target_expression(env, self.exc_value)
6646

6647 6648
        self.body = self.body.analyse_expressions(env)
        return self
6649

William Stein's avatar
William Stein committed
6650 6651 6652
    def generate_handling_code(self, code, end_label):
        code.mark_pos(self.pos)
        if self.pattern:
6653 6654 6655 6656 6657
            exc_tests = []
            for pattern in self.pattern:
                pattern.generate_evaluation_code(code)
                exc_tests.append("PyErr_ExceptionMatches(%s)" % pattern.py_result())

6658
            match_flag = code.funcstate.allocate_temp(PyrexTypes.c_int_type, False)
William Stein's avatar
William Stein committed
6659
            code.putln(
6660 6661 6662 6663
                "%s = %s;" % (match_flag, ' || '.join(exc_tests)))
            for pattern in self.pattern:
                pattern.generate_disposal_code(code)
                pattern.free_temps(code)
William Stein's avatar
William Stein committed
6664 6665
            code.putln(
                "if (%s) {" %
6666 6667
                    match_flag)
            code.funcstate.release_temp(match_flag)
William Stein's avatar
William Stein committed
6668
        else:
6669
            code.putln("/*except:*/ {")
6670

6671 6672
        if (not getattr(self.body, 'stats', True)
                and self.excinfo_target is None
6673
                and self.target is None):
Stefan Behnel's avatar
Stefan Behnel committed
6674 6675
            # most simple case: no exception variable, empty body (pass)
            # => reset the exception state, done
6676 6677 6678 6679
            code.putln("PyErr_Restore(0,0,0);")
            code.put_goto(end_label)
            code.putln("}")
            return
6680

6681 6682
        exc_vars = [code.funcstate.allocate_temp(py_object_type,
                                                 manage_ref=True)
Stefan Behnel's avatar
Stefan Behnel committed
6683
                    for _ in xrange(3)]
6684
        code.put_add_traceback(self.function_name)
William Stein's avatar
William Stein committed
6685
        # We always have to fetch the exception value even if
6686
        # there is no target, because this also normalises the
William Stein's avatar
William Stein committed
6687
        # exception and stores it in the thread state.
6688 6689
        code.globalstate.use_utility_code(get_exception_utility_code)
        exc_args = "&%s, &%s, &%s" % tuple(exc_vars)
6690 6691
        code.putln("if (__Pyx_GetException(%s) < 0) %s" % (exc_args,
            code.error_goto(self.pos)))
6692
        for x in exc_vars:
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
6693
            code.put_gotref(x)
William Stein's avatar
William Stein committed
6694
        if self.target:
6695
            self.exc_value.set_var(exc_vars[1])
6696
            self.exc_value.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
6697
            self.target.generate_assignment_code(self.exc_value, code)
6698
        if self.excinfo_target is not None:
6699
            for tempvar, node in zip(exc_vars, self.excinfo_target.args):
6700
                node.set_var(tempvar)
6701

6702 6703 6704 6705
        old_break_label, old_continue_label = code.break_label, code.continue_label
        code.break_label = code.new_label('except_break')
        code.continue_label = code.new_label('except_continue')

6706
        old_exc_vars = code.funcstate.exc_vars
6707
        code.funcstate.exc_vars = exc_vars
William Stein's avatar
William Stein committed
6708
        self.body.generate_execution_code(code)
6709
        code.funcstate.exc_vars = old_exc_vars
6710 6711 6712 6713
        if not self.body.is_terminator:
            for var in exc_vars:
                code.put_decref_clear(var, py_object_type)
            code.put_goto(end_label)
6714

Stefan Behnel's avatar
Stefan Behnel committed
6715 6716 6717 6718 6719 6720 6721
        for new_label, old_label in [(code.break_label, old_break_label),
                                     (code.continue_label, old_continue_label)]:
            if code.label_used(new_label):
                code.put_label(new_label)
                for var in exc_vars:
                    code.put_decref_clear(var, py_object_type)
                code.put_goto(old_label)
Robert Bradshaw's avatar
Robert Bradshaw committed
6722
        code.break_label = old_break_label
6723
        code.continue_label = old_continue_label
6724

6725 6726
        for temp in exc_vars:
            code.funcstate.release_temp(temp)
6727

William Stein's avatar
William Stein committed
6728 6729 6730
        code.putln(
            "}")

6731 6732 6733 6734
    def generate_function_definitions(self, env, code):
        if self.target is not None:
            self.target.generate_function_definitions(env, code)
        self.body.generate_function_definitions(env, code)
6735

6736
    def annotate(self, code):
6737
        if self.pattern:
6738 6739
            for pattern in self.pattern:
                pattern.annotate(code)
6740 6741 6742 6743
        if self.target:
            self.target.annotate(code)
        self.body.annotate(code)

William Stein's avatar
William Stein committed
6744 6745 6746 6747 6748 6749

class TryFinallyStatNode(StatNode):
    #  try ... finally statement
    #
    #  body             StatNode
    #  finally_clause   StatNode
6750
    #
William Stein's avatar
William Stein committed
6751 6752 6753 6754 6755 6756 6757 6758
    #  The plan is that we funnel all continue, break
    #  return and error gotos into the beginning of the
    #  finally block, setting a variable to remember which
    #  one we're doing. At the end of the finally block, we
    #  switch on the variable to figure out where to go.
    #  In addition, if we're doing an error, we save the
    #  exception on entry to the finally block and restore
    #  it on exit.
6759

6760
    child_attrs = ["body", "finally_clause"]
6761

6762
    preserve_exception = 1
6763

6764 6765
    # handle exception case, in addition to return/break/continue
    handle_error_case = True
6766
    func_return_type = None
6767

William Stein's avatar
William Stein committed
6768 6769 6770 6771
    disallow_continue_in_try_finally = 0
    # There doesn't seem to be any point in disallowing
    # continue in the try block, since we have no problem
    # handling it.
6772

6773 6774
    is_try_finally_in_nogil = False

6775 6776 6777 6778
    def create_analysed(pos, env, body, finally_clause):
        node = TryFinallyStatNode(pos, body=body, finally_clause=finally_clause)
        return node
    create_analysed = staticmethod(create_analysed)
6779

William Stein's avatar
William Stein committed
6780 6781 6782
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)
        self.finally_clause.analyse_declarations(env)
6783

William Stein's avatar
William Stein committed
6784
    def analyse_expressions(self, env):
6785 6786
        self.body = self.body.analyse_expressions(env)
        self.finally_clause = self.finally_clause.analyse_expressions(env)
Stefan Behnel's avatar
Stefan Behnel committed
6787 6788
        if env.return_type and not env.return_type.is_void:
            self.func_return_type = env.return_type
6789
        return self
6790

6791
    nogil_check = Node.gil_error
6792 6793
    gil_message = "Try-finally statement"

William Stein's avatar
William Stein committed
6794 6795 6796 6797 6798
    def generate_execution_code(self, code):
        old_error_label = code.error_label
        old_labels = code.all_new_labels()
        new_labels = code.get_all_labels()
        new_error_label = code.error_label
6799 6800
        if not self.handle_error_case:
            code.error_label = old_error_label
William Stein's avatar
William Stein committed
6801
        catch_label = code.new_label()
6802 6803 6804

        code.putln("/*try:*/ {")

William Stein's avatar
William Stein committed
6805
        if self.disallow_continue_in_try_finally:
6806 6807
            was_in_try_finally = code.funcstate.in_try_finally
            code.funcstate.in_try_finally = 1
6808

William Stein's avatar
William Stein committed
6809
        self.body.generate_execution_code(code)
6810

William Stein's avatar
William Stein committed
6811
        if self.disallow_continue_in_try_finally:
6812
            code.funcstate.in_try_finally = was_in_try_finally
6813 6814

        code.putln("}")
6815
        code.set_all_labels(old_labels)
6816

6817
        temps_to_clean_up = code.funcstate.all_free_managed_temps()
6818
        code.mark_pos(self.finally_clause.pos)
6819 6820
        code.putln("/*finally:*/ {")

6821 6822 6823 6824 6825 6826 6827 6828 6829
        def fresh_finally_clause(_next=[self.finally_clause]):
            # generate the original subtree once and always keep a fresh copy
            node = _next[0]
            node_copy = copy.deepcopy(node)
            if node is self.finally_clause:
                _next[0] = node_copy
            else:
                node = node_copy
            return node
6830 6831

        preserve_error = self.preserve_exception and code.label_used(new_error_label)
6832
        needs_success_cleanup = not self.finally_clause.is_terminator
6833 6834 6835

        if not self.body.is_terminator:
            code.putln('/*normal exit:*/{')
6836 6837
            fresh_finally_clause().generate_execution_code(code)
            if not self.finally_clause.is_terminator:
6838
                code.put_goto(catch_label)
6839
            code.putln('}')
6840

6841 6842
        if preserve_error:
            code.putln('/*exception exit:*/{')
6843 6844
            if self.is_try_finally_in_nogil:
                code.declare_gilstate()
6845 6846 6847 6848 6849 6850 6851 6852 6853 6854 6855
            if needs_success_cleanup:
                exc_lineno_cnames = tuple([
                    code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
                    for _ in range(2)])
                exc_filename_cname = code.funcstate.allocate_temp(
                    PyrexTypes.CPtrType(PyrexTypes.c_const_type(PyrexTypes.c_char_type)),
                    manage_ref=False)
            else:
                exc_lineno_cnames = exc_filename_cname = None
            exc_vars = tuple([
                code.funcstate.allocate_temp(py_object_type, manage_ref=False)
6856
                for _ in range(6)])
6857
            code.put_label(new_error_label)
6858
            self.put_error_catcher(
6859
                code, temps_to_clean_up, exc_vars, exc_lineno_cnames, exc_filename_cname)
6860
            finally_old_labels = code.all_new_labels()
6861

6862
            code.putln('{')
6863 6864
            old_exc_vars = code.funcstate.exc_vars
            code.funcstate.exc_vars = exc_vars[:3]
6865
            fresh_finally_clause().generate_execution_code(code)
6866
            code.funcstate.exc_vars = old_exc_vars
6867 6868
            code.putln('}')

6869
            if needs_success_cleanup:
6870 6871 6872 6873 6874 6875
                self.put_error_uncatcher(code, exc_vars, exc_lineno_cnames, exc_filename_cname)
                if exc_lineno_cnames:
                    for cname in exc_lineno_cnames:
                        code.funcstate.release_temp(cname)
                if exc_filename_cname:
                    code.funcstate.release_temp(exc_filename_cname)
6876
                code.put_goto(old_error_label)
6877

6878 6879 6880 6881
            for new_label, old_label in zip(code.get_all_labels(), finally_old_labels):
                if not code.label_used(new_label):
                    continue
                code.put_label(new_label)
6882
                self.put_error_cleaner(code, exc_vars)
6883
                code.put_goto(old_label)
6884 6885 6886

            for cname in exc_vars:
                code.funcstate.release_temp(cname)
6887
            code.putln('}')
6888

6889
        code.set_all_labels(old_labels)
6890
        return_label = code.return_label
6891 6892 6893 6894 6895
        for i, (new_label, old_label) in enumerate(zip(new_labels, old_labels)):
            if not code.label_used(new_label):
                continue
            if new_label == new_error_label and preserve_error:
                continue  # handled above
6896

6897 6898
            code.put('%s: ' % new_label)
            code.putln('{')
6899 6900 6901
            ret_temp = None
            if old_label == return_label and not self.finally_clause.is_terminator:
                # store away return value for later reuse
6902 6903 6904
                if (self.func_return_type and
                        not self.is_try_finally_in_nogil and
                        not isinstance(self.finally_clause, GILExitNode)):
6905 6906 6907 6908 6909
                    ret_temp = code.funcstate.allocate_temp(
                        self.func_return_type, manage_ref=False)
                    code.putln("%s = %s;" % (ret_temp, Naming.retval_cname))
                    if self.func_return_type.is_pyobject:
                        code.putln("%s = 0;" % Naming.retval_cname)
6910
            fresh_finally_clause().generate_execution_code(code)
6911 6912 6913 6914 6915 6916
            if ret_temp:
                code.putln("%s = %s;" % (Naming.retval_cname, ret_temp))
                if self.func_return_type.is_pyobject:
                    code.putln("%s = 0;" % ret_temp)
                code.funcstate.release_temp(ret_temp)
                ret_temp = None
6917
            if not self.finally_clause.is_terminator:
6918 6919
                code.put_goto(old_label)
            code.putln('}')
6920 6921

        # End finally
6922
        code.put_label(catch_label)
William Stein's avatar
William Stein committed
6923 6924 6925
        code.putln(
            "}")

6926 6927 6928 6929
    def generate_function_definitions(self, env, code):
        self.body.generate_function_definitions(env, code)
        self.finally_clause.generate_function_definitions(env, code)

6930 6931
    def put_error_catcher(self, code, temps_to_clean_up, exc_vars,
                          exc_lineno_cnames, exc_filename_cname):
6932
        code.globalstate.use_utility_code(restore_exception_utility_code)
6933 6934
        code.globalstate.use_utility_code(get_exception_utility_code)
        code.globalstate.use_utility_code(swap_exception_utility_code)
6935

6936
        code.putln(' '.join(["%s = 0;"]*len(exc_vars)) % exc_vars)
6937 6938 6939
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)

6940 6941
        for temp_name, type in temps_to_clean_up:
            code.put_xdecref_clear(temp_name, type)
6942

6943 6944 6945 6946 6947 6948 6949 6950 6951
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3)"
                   " __Pyx_ExceptionSwap(&%s, &%s, &%s);" % exc_vars[3:])
        code.putln("if ((PY_MAJOR_VERSION < 3) ||"
                   # if __Pyx_GetException() fails in Py3,
                   # store the newly raised exception instead
                   " unlikely(__Pyx_GetException(&%s, &%s, &%s) < 0)) "
                   "__Pyx_ErrFetch(&%s, &%s, &%s);" % (exc_vars[:3] * 2))
6952 6953 6954 6955 6956 6957 6958
        for var in exc_vars:
            code.put_xgotref(var)
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                exc_lineno_cnames[0], Naming.lineno_cname,
                exc_lineno_cnames[1], Naming.clineno_cname,
                exc_filename_cname, Naming.filename_cname))
6959 6960 6961 6962

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

6963
    def put_error_uncatcher(self, code, exc_vars, exc_lineno_cnames, exc_filename_cname):
6964
        code.globalstate.use_utility_code(restore_exception_utility_code)
6965
        code.globalstate.use_utility_code(reset_exception_utility_code)
6966 6967 6968 6969

        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)

6970 6971 6972
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
6973 6974 6975
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
6976
        code.putln("}")
6977
        for var in exc_vars[:3]:
6978
            code.put_xgiveref(var)
6979
        code.putln("__Pyx_ErrRestore(%s, %s, %s);" % exc_vars[:3])
6980 6981 6982 6983

        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()

6984
        code.putln(' '.join(["%s = 0;"]*len(exc_vars)) % exc_vars)
6985 6986 6987 6988 6989
        if exc_lineno_cnames:
            code.putln("%s = %s; %s = %s; %s = %s;" % (
                Naming.lineno_cname, exc_lineno_cnames[0],
                Naming.clineno_cname, exc_lineno_cnames[1],
                Naming.filename_cname, exc_filename_cname))
6990

6991
    def put_error_cleaner(self, code, exc_vars):
6992
        code.globalstate.use_utility_code(reset_exception_utility_code)
6993 6994
        if self.is_try_finally_in_nogil:
            code.put_ensure_gil(declare_gilstate=False)
6995 6996 6997
        # not using preprocessor here to avoid warnings about
        # unused utility functions and/or temps
        code.putln("if (PY_MAJOR_VERSION >= 3) {")
6998 6999 7000
        for var in exc_vars[3:]:
            code.put_xgiveref(var)
        code.putln("__Pyx_ExceptionReset(%s, %s, %s);" % exc_vars[3:])
7001
        code.putln("}")
7002
        for var in exc_vars[:3]:
7003
            code.put_xdecref_clear(var, py_object_type)
7004 7005
        if self.is_try_finally_in_nogil:
            code.put_release_ensured_gil()
7006
        code.putln(' '.join(["%s = 0;"]*3) % exc_vars[3:])
William Stein's avatar
William Stein committed
7007

7008 7009 7010 7011
    def annotate(self, code):
        self.body.annotate(code)
        self.finally_clause.annotate(code)

William Stein's avatar
William Stein committed
7012

7013 7014 7015 7016 7017 7018 7019 7020 7021 7022
class NogilTryFinallyStatNode(TryFinallyStatNode):
    """
    A try/finally statement that may be used in nogil code sections.
    """

    preserve_exception = False
    nogil_check = None


class GILStatNode(NogilTryFinallyStatNode):
7023 7024 7025
    #  'with gil' or 'with nogil' statement
    #
    #   state   string   'gil' or 'nogil'
7026

7027 7028
    state_temp = None

7029 7030
    def __init__(self, pos, state, body):
        self.state = state
7031 7032 7033 7034 7035 7036 7037
        self.create_state_temp_if_needed(pos, state, body)
        TryFinallyStatNode.__init__(self, pos,
            body=body,
            finally_clause=GILExitNode(
                pos, state=state, state_temp=self.state_temp))

    def create_state_temp_if_needed(self, pos, state, body):
7038
        from .ParseTreeTransforms import YieldNodeCollector
7039 7040 7041 7042 7043
        collector = YieldNodeCollector()
        collector.visitchildren(body)
        if not collector.yields:
            return

7044 7045 7046 7047
        if state == 'gil':
            temp_type = PyrexTypes.c_gilstate_type
        else:
            temp_type = PyrexTypes.c_threadstate_ptr_type
7048
        from . import ExprNodes
7049
        self.state_temp = ExprNodes.TempNode(pos, temp_type)
7050

7051 7052 7053 7054
    def analyse_declarations(self, env):
        env._in_with_gil_block = (self.state == 'gil')
        if self.state == 'gil':
            env.has_with_gil_block = True
7055

7056 7057
        return super(GILStatNode, self).analyse_declarations(env)

7058
    def analyse_expressions(self, env):
7059 7060
        env.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
7061
        was_nogil = env.nogil
7062
        env.nogil = self.state == 'nogil'
7063
        node = TryFinallyStatNode.analyse_expressions(self, env)
7064
        env.nogil = was_nogil
7065
        return node
7066

7067
    def generate_execution_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
7068
        code.mark_pos(self.pos)
7069
        code.begin_block()
7070 7071 7072 7073 7074
        if self.state_temp:
            self.state_temp.allocate(code)
            variable = self.state_temp.result()
        else:
            variable = None
7075

7076
        old_trace_config = code.funcstate.can_trace
7077
        if self.state == 'gil':
7078
            code.put_ensure_gil(variable=variable)
7079 7080
            # FIXME: not that easy, tracing may not be possible at all here
            #code.funcstate.can_trace = True
7081
        else:
7082
            code.put_release_gil(variable=variable)
7083
            code.funcstate.can_trace = False
7084

7085
        TryFinallyStatNode.generate_execution_code(self, code)
7086

7087 7088
        if self.state_temp:
            self.state_temp.release(code)
7089 7090

        code.funcstate.can_trace = old_trace_config
7091
        code.end_block()
7092 7093 7094


class GILExitNode(StatNode):
7095 7096 7097 7098 7099
    """
    Used as the 'finally' block in a GILStatNode

    state   string   'gil' or 'nogil'
    """
7100

7101
    child_attrs = []
7102
    state_temp = None
7103

7104
    def analyse_expressions(self, env):
7105
        return self
7106 7107

    def generate_execution_code(self, code):
7108 7109 7110 7111 7112
        if self.state_temp:
            variable = self.state_temp.result()
        else:
            variable = None

7113
        if self.state == 'gil':
7114
            code.put_release_ensured_gil(variable)
7115
        else:
7116
            code.put_acquire_gil(variable)
7117 7118


7119 7120 7121 7122 7123 7124 7125
class EnsureGILNode(GILExitNode):
    """
    Ensure the GIL in nogil functions for cleanup before returning.
    """

    def generate_execution_code(self, code):
        code.put_ensure_gil(declare_gilstate=False)
7126

7127

7128 7129 7130 7131 7132 7133
utility_code_for_cimports = {
    # utility code (or inlining c) in a pxd (or pyx) file.
    # TODO: Consider a generic user-level mechanism for importing
    'cpython.array'         : ("ArrayAPI", "arrayarray.h"),
    'cpython.array.array'   : ("ArrayAPI", "arrayarray.h"),
}
7134

7135

William Stein's avatar
William Stein committed
7136 7137 7138 7139 7140
class CImportStatNode(StatNode):
    #  cimport statement
    #
    #  module_name   string           Qualified name of module being imported
    #  as_name       string or None   Name specified in "as" clause, if any
7141
    #  is_absolute   bool             True for absolute imports, False otherwise
7142 7143

    child_attrs = []
7144
    is_absolute = False
7145

William Stein's avatar
William Stein committed
7146
    def analyse_declarations(self, env):
7147 7148 7149
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
7150 7151
        module_scope = env.find_module(
            self.module_name, self.pos, relative_level=0 if self.is_absolute else -1)
William Stein's avatar
William Stein committed
7152
        if "." in self.module_name:
7153
            names = [EncodedString(name) for name in self.module_name.split(".")]
William Stein's avatar
William Stein committed
7154 7155 7156 7157 7158 7159 7160 7161 7162 7163
            top_name = names[0]
            top_module_scope = env.context.find_submodule(top_name)
            module_scope = top_module_scope
            for name in names[1:]:
                submodule_scope = module_scope.find_submodule(name)
                module_scope.declare_module(name, submodule_scope, self.pos)
                module_scope = submodule_scope
            if self.as_name:
                env.declare_module(self.as_name, module_scope, self.pos)
            else:
7164
                env.add_imported_module(module_scope)
William Stein's avatar
William Stein committed
7165 7166 7167 7168
                env.declare_module(top_name, top_module_scope, self.pos)
        else:
            name = self.as_name or self.module_name
            env.declare_module(name, module_scope, self.pos)
7169 7170 7171
        if self.module_name in utility_code_for_cimports:
            env.use_utility_code(UtilityCode.load_cached(
                *utility_code_for_cimports[self.module_name]))
William Stein's avatar
William Stein committed
7172 7173

    def analyse_expressions(self, env):
7174
        return self
7175

William Stein's avatar
William Stein committed
7176 7177
    def generate_execution_code(self, code):
        pass
7178

William Stein's avatar
William Stein committed
7179 7180 7181 7182

class FromCImportStatNode(StatNode):
    #  from ... cimport statement
    #
7183
    #  module_name     string                        Qualified name of module
7184
    #  relative_level  int or None                   Relative import: number of dots before module_name
7185
    #  imported_names  [(pos, name, as_name, kind)]  Names to be imported
7186

7187
    child_attrs = []
7188 7189 7190
    module_name = None
    relative_level = None
    imported_names = None
7191

William Stein's avatar
William Stein committed
7192
    def analyse_declarations(self, env):
7193 7194 7195
        if not env.is_module_scope:
            error(self.pos, "cimport only allowed at module level")
            return
7196 7197 7198 7199
        if self.relative_level and self.relative_level > env.qualified_name.count('.'):
            error(self.pos, "relative cimport beyond main package is not allowed")
        module_scope = env.find_module(self.module_name, self.pos, relative_level=self.relative_level)
        module_name = module_scope.qualified_name
William Stein's avatar
William Stein committed
7200
        env.add_imported_module(module_scope)
7201
        for pos, name, as_name, kind in self.imported_names:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7202 7203 7204 7205
            if name == "*":
                for local_name, entry in module_scope.entries.items():
                    env.add_imported_entry(local_name, entry, pos)
            else:
7206 7207 7208 7209
                entry = module_scope.lookup(name)
                if entry:
                    if kind and not self.declaration_matches(entry, kind):
                        entry.redeclared(pos)
7210
                    entry.used = 1
7211 7212
                else:
                    if kind == 'struct' or kind == 'union':
7213 7214
                        entry = module_scope.declare_struct_or_union(
                            name, kind=kind, scope=None, typedef_flag=0, pos=pos)
7215
                    elif kind == 'class':
7216
                        entry = module_scope.declare_c_class(name, pos=pos, module_name=module_name)
7217
                    else:
7218
                        submodule_scope = env.context.find_module(name, relative_to=module_scope, pos=self.pos)
7219 7220 7221
                        if submodule_scope.parent_module is module_scope:
                            env.declare_module(as_name or name, submodule_scope, self.pos)
                        else:
7222
                            error(pos, "Name '%s' not declared in module '%s'" % (name, module_name))
7223

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7224 7225 7226
                if entry:
                    local_name = as_name or name
                    env.add_imported_entry(local_name, entry, pos)
7227

7228 7229
        if module_name.startswith('cpython'): # enough for now
            if module_name in utility_code_for_cimports:
7230
                env.use_utility_code(UtilityCode.load_cached(
7231
                    *utility_code_for_cimports[module_name]))
7232
            for _, name, _, _ in self.imported_names:
7233
                fqname = '%s.%s' % (module_name, name)
7234 7235 7236
                if fqname in utility_code_for_cimports:
                    env.use_utility_code(UtilityCode.load_cached(
                        *utility_code_for_cimports[fqname]))
7237

7238
    def declaration_matches(self, entry, kind):
7239 7240 7241 7242 7243 7244 7245 7246 7247 7248 7249 7250
        if not entry.is_type:
            return 0
        type = entry.type
        if kind == 'class':
            if not type.is_extension_type:
                return 0
        else:
            if not type.is_struct_or_union:
                return 0
            if kind != type.kind:
                return 0
        return 1
William Stein's avatar
William Stein committed
7251 7252

    def analyse_expressions(self, env):
7253
        return self
7254

William Stein's avatar
William Stein committed
7255 7256 7257 7258 7259 7260 7261 7262 7263
    def generate_execution_code(self, code):
        pass


class FromImportStatNode(StatNode):
    #  from ... import statement
    #
    #  module           ImportNode
    #  items            [(string, NameNode)]
7264
    #  interned_items   [(string, NameNode, ExprNode)]
William Stein's avatar
William Stein committed
7265
    #  item             PyTempNode            used internally
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7266
    #  import_star      boolean               used internally
7267 7268

    child_attrs = ["module"]
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7269
    import_star = 0
7270

William Stein's avatar
William Stein committed
7271
    def analyse_declarations(self, env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7272 7273 7274 7275 7276 7277 7278 7279 7280
        for name, target in self.items:
            if name == "*":
                if not env.is_module_scope:
                    error(self.pos, "import * only allowed at module level")
                    return
                env.has_import_star = 1
                self.import_star = 1
            else:
                target.analyse_target_declaration(env)
7281

William Stein's avatar
William Stein committed
7282
    def analyse_expressions(self, env):
7283
        from . import ExprNodes
7284
        self.module = self.module.analyse_expressions(env)
7285
        self.item = ExprNodes.RawCNameExprNode(self.pos, py_object_type)
William Stein's avatar
William Stein committed
7286 7287
        self.interned_items = []
        for name, target in self.items:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7288 7289 7290
            if name == '*':
                for _, entry in env.entries.items():
                    if not entry.is_type and entry.type.is_extension_type:
7291
                        env.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7292 7293
                        break
            else:
7294
                entry = env.lookup(target.name)
7295 7296
                # check whether or not entry is already cimported
                if (entry.is_type and entry.type.name == name
Stefan Behnel's avatar
Stefan Behnel committed
7297
                        and hasattr(entry.type, 'module_name')):
7298 7299 7300 7301 7302
                    if entry.type.module_name == self.module.module_name.value:
                        # cimported with absolute name
                        continue
                    try:
                        # cimported with relative name
7303 7304
                        module = env.find_module(self.module.module_name.value, pos=self.pos,
                                                 relative_level=self.module.level)
7305 7306 7307
                        if entry.type.module_name == module.qualified_name:
                            continue
                    except AttributeError:
7308
                        pass
7309
                target = target.analyse_target_expression(env, None)  # FIXME?
7310 7311 7312 7313
                if target.type is py_object_type:
                    coerced_item = None
                else:
                    coerced_item = self.item.coerce_to(target.type, env)
7314
                self.interned_items.append((name, target, coerced_item))
7315
        return self
7316

William Stein's avatar
William Stein committed
7317 7318
    def generate_execution_code(self, code):
        self.module.generate_evaluation_code(code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7319 7320 7321 7322 7323 7324
        if self.import_star:
            code.putln(
                'if (%s(%s) < 0) %s;' % (
                    Naming.import_star,
                    self.module.py_result(),
                    code.error_goto(self.pos)))
7325 7326
        item_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        self.item.set_cname(item_temp)
7327 7328 7329
        if self.interned_items:
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("ImportFrom", "ImportExport.c"))
7330
        for name, target, coerced_item in self.interned_items:
7331
            code.putln(
7332
                '%s = __Pyx_ImportFrom(%s, %s); %s' % (
7333
                    item_temp,
7334
                    self.module.py_result(),
7335 7336
                    code.intern_identifier(name),
                    code.error_goto_if_null(item_temp, self.pos)))
7337
            code.put_gotref(item_temp)
7338 7339 7340 7341 7342 7343
            if coerced_item is None:
                target.generate_assignment_code(self.item, code)
            else:
                coerced_item.allocate_temp_result(code)
                coerced_item.generate_result_code(code)
                target.generate_assignment_code(coerced_item, code)
7344 7345
            code.put_decref_clear(item_temp, py_object_type)
        code.funcstate.release_temp(item_temp)
William Stein's avatar
William Stein committed
7346
        self.module.generate_disposal_code(code)
7347
        self.module.free_temps(code)
William Stein's avatar
William Stein committed
7348

7349

Mark Florisson's avatar
Mark Florisson committed
7350 7351 7352 7353 7354 7355 7356 7357 7358 7359
class ParallelNode(Node):
    """
    Base class for cython.parallel constructs.
    """

    nogil_check = None


class ParallelStatNode(StatNode, ParallelNode):
    """
7360
    Base class for 'with cython.parallel.parallel():' and 'for i in prange():'.
Mark Florisson's avatar
Mark Florisson committed
7361 7362 7363 7364 7365

    assignments     { Entry(var) : (var.pos, inplace_operator_or_None) }
                    assignments to variables in this parallel section

    parent          parent ParallelStatNode or None
7366 7367 7368
    is_parallel     indicates whether this node is OpenMP parallel
                    (true for #pragma omp parallel for and
                              #pragma omp parallel)
Mark Florisson's avatar
Mark Florisson committed
7369 7370 7371 7372 7373 7374 7375 7376 7377 7378 7379

    is_parallel is true for:

        #pragma omp parallel
        #pragma omp parallel for

    sections, but NOT for

        #pragma omp for

    We need this to determine the sharing attributes.
7380 7381 7382

    privatization_insertion_point   a code insertion point used to make temps
                                    private (esp. the "nsteps" temp)
7383 7384 7385 7386

    args         tuple          the arguments passed to the parallel construct
    kwargs       DictNode       the keyword arguments passed to the parallel
                                construct (replaced by its compile time value)
Mark Florisson's avatar
Mark Florisson committed
7387 7388
    """

7389
    child_attrs = ['body', 'num_threads']
Mark Florisson's avatar
Mark Florisson committed
7390 7391 7392 7393

    body = None

    is_prange = False
7394
    is_nested_prange = False
Mark Florisson's avatar
Mark Florisson committed
7395

7396
    error_label_used = False
7397

7398
    num_threads = None
7399
    chunksize = None
7400

7401 7402 7403 7404 7405 7406 7407 7408 7409 7410 7411 7412 7413 7414 7415 7416 7417
    parallel_exc = (
        Naming.parallel_exc_type,
        Naming.parallel_exc_value,
        Naming.parallel_exc_tb,
    )

    parallel_pos_info = (
        Naming.parallel_filename,
        Naming.parallel_lineno,
        Naming.parallel_clineno,
    )

    pos_info = (
        Naming.filename_cname,
        Naming.lineno_cname,
        Naming.clineno_cname,
    )
7418

7419 7420
    critical_section_counter = 0

Mark Florisson's avatar
Mark Florisson committed
7421 7422
    def __init__(self, pos, **kwargs):
        super(ParallelStatNode, self).__init__(pos, **kwargs)
7423 7424

        # All assignments in this scope
Mark Florisson's avatar
Mark Florisson committed
7425 7426
        self.assignments = kwargs.get('assignments') or {}

7427 7428 7429 7430
        # All seen closure cnames and their temporary cnames
        self.seen_closure_vars = set()

        # Dict of variables that should be declared (first|last|)private or
7431 7432
        # reduction { Entry: (op, lastprivate) }.
        # If op is not None, it's a reduction.
7433
        self.privates = {}
Mark Florisson's avatar
Mark Florisson committed
7434

Mark Florisson's avatar
Mark Florisson committed
7435 7436 7437
        # [NameNode]
        self.assigned_nodes = []

Mark Florisson's avatar
Mark Florisson committed
7438 7439 7440
    def analyse_declarations(self, env):
        self.body.analyse_declarations(env)

7441 7442
        self.num_threads = None

7443
        if self.kwargs:
7444 7445 7446
            # Try to find num_threads and chunksize keyword arguments
            pairs = []
            for dictitem in self.kwargs.key_value_pairs:
7447 7448
                if dictitem.key.value == 'num_threads':
                    self.num_threads = dictitem.value
7449 7450 7451 7452 7453 7454
                elif self.is_prange and dictitem.key.value == 'chunksize':
                    self.chunksize = dictitem.value
                else:
                    pairs.append(dictitem)

            self.kwargs.key_value_pairs = pairs
7455

7456 7457 7458 7459 7460
            try:
                self.kwargs = self.kwargs.compile_time_value(env)
            except Exception, e:
                error(self.kwargs.pos, "Only compile-time values may be "
                                       "supplied as keyword arguments")
7461 7462 7463 7464 7465 7466 7467 7468 7469
        else:
            self.kwargs = {}

        for kw, val in self.kwargs.iteritems():
            if kw not in self.valid_keyword_arguments:
                error(self.pos, "Invalid keyword argument: %s" % kw)
            else:
                setattr(self, kw, val)

7470
    def analyse_expressions(self, env):
7471
        if self.num_threads:
7472
            self.num_threads = self.num_threads.analyse_expressions(env)
7473 7474

        if self.chunksize:
7475
            self.chunksize = self.chunksize.analyse_expressions(env)
7476

7477
        self.body = self.body.analyse_expressions(env)
7478
        self.analyse_sharing_attributes(env)
7479

7480
        if self.num_threads is not None:
7481 7482
            if (self.parent and self.parent.num_threads is not None and not
                                                    self.parent.is_prange):
7483 7484
                error(self.pos,
                      "num_threads already declared in outer section")
7485
            elif self.parent and not self.parent.is_prange:
7486
                error(self.pos,
7487 7488 7489 7490
                      "num_threads must be declared in the parent parallel section")
            elif (self.num_threads.type.is_int and
                  self.num_threads.is_literal and
                  self.num_threads.compile_time_value(env) <= 0):
7491 7492 7493
                error(self.pos,
                      "argument to num_threads must be greater than 0")

Mark Florisson's avatar
Mark Florisson committed
7494 7495
            if not self.num_threads.is_simple():
                self.num_threads = self.num_threads.coerce_to(
7496 7497
                    PyrexTypes.c_int_type, env).coerce_to_temp(env)
        return self
7498 7499

    def analyse_sharing_attributes(self, env):
Mark Florisson's avatar
Mark Florisson committed
7500
        """
7501 7502 7503
        Analyse the privates for this block and set them in self.privates.
        This should be called in a post-order fashion during the
        analyse_expressions phase
Mark Florisson's avatar
Mark Florisson committed
7504
        """
7505
        for entry, (pos, op) in self.assignments.iteritems():
Mark Florisson's avatar
Mark Florisson committed
7506

7507 7508 7509 7510 7511 7512
            if self.is_prange and not self.is_parallel:
                # closely nested prange in a with parallel block, disallow
                # assigning to privates in the with parallel block (we
                # consider it too implicit and magicky for users)
                if entry in self.parent.assignments:
                    error(pos,
Mark Florisson's avatar
Mark Florisson committed
7513
                          "Cannot assign to private of outer parallel block")
7514 7515 7516 7517 7518 7519 7520
                    continue

            if not self.is_prange and op:
                # Again possible, but considered to magicky
                error(pos, "Reductions not allowed for parallel blocks")
                continue

Mark Florisson's avatar
Mark Florisson committed
7521 7522 7523
            # By default all variables should have the same values as if
            # executed sequentially
            lastprivate = True
7524
            self.propagate_var_privatization(entry, pos, op, lastprivate)
7525

7526
    def propagate_var_privatization(self, entry, pos, op, lastprivate):
7527 7528 7529 7530 7531 7532 7533 7534 7535 7536 7537 7538 7539 7540 7541 7542 7543 7544 7545 7546 7547 7548 7549 7550 7551 7552 7553 7554
        """
        Propagate the sharing attributes of a variable. If the privatization is
        determined by a parent scope, done propagate further.

        If we are a prange, we propagate our sharing attributes outwards to
        other pranges. If we are a prange in parallel block and the parallel
        block does not determine the variable private, we propagate to the
        parent of the parent. Recursion stops at parallel blocks, as they have
        no concept of lastprivate or reduction.

        So the following cases propagate:

            sum is a reduction for all loops:

                for i in prange(n):
                    for j in prange(n):
                        for k in prange(n):
                            sum += i * j * k

            sum is a reduction for both loops, local_var is private to the
            parallel with block:

                for i in prange(n):
                    with parallel:
                        local_var = ... # private to the parallel
                        for j in prange(n):
                            sum += i * j

Mark Florisson's avatar
Mark Florisson committed
7555 7556
        Nested with parallel blocks are disallowed, because they wouldn't
        allow you to propagate lastprivates or reductions:
7557 7558 7559 7560

            #pragma omp parallel for lastprivate(i)
            for i in prange(n):

Mark Florisson's avatar
Mark Florisson committed
7561 7562
                sum = 0

7563 7564 7565 7566 7567 7568 7569 7570 7571 7572 7573 7574 7575 7576 7577 7578
                #pragma omp parallel private(j, sum)
                with parallel:

                    #pragma omp parallel
                    with parallel:

                        #pragma omp for lastprivate(j) reduction(+:sum)
                        for j in prange(n):
                            sum += i

                    # sum and j are well-defined here

                # sum and j are undefined here

            # sum and j are undefined here
        """
7579
        self.privates[entry] = (op, lastprivate)
7580 7581 7582 7583 7584

        if entry.type.is_memoryviewslice:
            error(pos, "Memoryview slices can only be shared in parallel sections")
            return

7585 7586 7587 7588 7589 7590
        if self.is_prange:
            if not self.is_parallel and entry not in self.parent.assignments:
                # Parent is a parallel with block
                parent = self.parent.parent
            else:
                parent = self.parent
Mark Florisson's avatar
Mark Florisson committed
7591

7592 7593 7594
            # We don't need to propagate privates, only reductions and
            # lastprivates
            if parent and (op or lastprivate):
7595
                parent.propagate_var_privatization(entry, pos, op, lastprivate)
Mark Florisson's avatar
Mark Florisson committed
7596 7597 7598 7599 7600 7601 7602 7603 7604

    def _allocate_closure_temp(self, code, entry):
        """
        Helper function that allocate a temporary for a closure variable that
        is assigned to.
        """
        if self.parent:
            return self.parent._allocate_closure_temp(code, entry)

7605 7606 7607
        if entry.cname in self.seen_closure_vars:
            return entry.cname

7608
        cname = code.funcstate.allocate_temp(entry.type, True)
7609 7610 7611 7612 7613 7614

        # Add both the actual cname and the temp cname, as the actual cname
        # will be replaced with the temp cname on the entry
        self.seen_closure_vars.add(entry.cname)
        self.seen_closure_vars.add(cname)

Mark Florisson's avatar
Mark Florisson committed
7615 7616 7617 7618
        self.modified_entries.append((entry, entry.cname))
        code.putln("%s = %s;" % (cname, entry.cname))
        entry.cname = cname

7619
    def initialize_privates_to_nan(self, code, exclude=None):
7620
        first = True
7621

7622
        for entry, (op, lastprivate) in self.privates.iteritems():
7623
            if not op and (not exclude or entry != exclude):
7624
                invalid_value = entry.type.invalid_value()
7625

7626
                if invalid_value:
7627 7628 7629 7630
                    if first:
                        code.putln("/* Initialize private variables to "
                                   "invalid values */")
                        first = False
7631
                    code.putln("%s = %s;" % (entry.cname,
7632
                                             entry.type.cast_code(invalid_value)))
7633

7634
    def evaluate_before_block(self, code, expr):
Mark Florisson's avatar
Mark Florisson committed
7635
        c = self.begin_of_parallel_control_block_point_after_decls
7636 7637 7638 7639 7640 7641 7642 7643 7644 7645
        # we need to set the owner to ourselves temporarily, as
        # allocate_temp may generate a comment in the middle of our pragma
        # otherwise when DebugFlags.debug_temp_code_comments is in effect
        owner = c.funcstate.owner
        c.funcstate.owner = c
        expr.generate_evaluation_code(c)
        c.funcstate.owner = owner

        return expr.result()

7646 7647 7648 7649 7650
    def put_num_threads(self, code):
        """
        Write self.num_threads if set as the num_threads OpenMP directive
        """
        if self.num_threads is not None:
7651 7652
            code.put(" num_threads(%s)" % self.evaluate_before_block(code,
                                                        self.num_threads))
7653

7654

Mark Florisson's avatar
Mark Florisson committed
7655 7656 7657 7658 7659 7660 7661 7662 7663
    def declare_closure_privates(self, code):
        """
        If a variable is in a scope object, we need to allocate a temp and
        assign the value from the temp to the variable in the scope object
        after the parallel section. This kind of copying should be done only
        in the outermost parallel section.
        """
        self.modified_entries = []

Stefan Behnel's avatar
Stefan Behnel committed
7664
        for entry in self.assignments:
Mark Florisson's avatar
Mark Florisson committed
7665
            if entry.from_closure or entry.in_closure:
7666
                self._allocate_closure_temp(code, entry)
Mark Florisson's avatar
Mark Florisson committed
7667 7668

    def release_closure_privates(self, code):
7669 7670 7671
        """
        Release any temps used for variables in scope objects. As this is the
        outermost parallel block, we don't need to delete the cnames from
7672
        self.seen_closure_vars.
7673
        """
Mark Florisson's avatar
Mark Florisson committed
7674 7675 7676 7677 7678
        for entry, original_cname in self.modified_entries:
            code.putln("%s = %s;" % (original_cname, entry.cname))
            code.funcstate.release_temp(entry.cname)
            entry.cname = original_cname

7679 7680 7681 7682 7683 7684
    def privatize_temps(self, code, exclude_temps=()):
        """
        Make any used temporaries private. Before the relevant code block
        code.start_collecting_temps() should have been called.
        """
        if self.is_parallel:
7685
            c = self.privatization_insertion_point
7686

7687
            self.temps = temps = code.funcstate.stop_collecting_temps()
7688 7689
            privates, firstprivates = [], []
            for temp, type in temps:
7690
                if type.is_pyobject or type.is_memoryviewslice:
7691 7692 7693
                    firstprivates.append(temp)
                else:
                    privates.append(temp)
7694

7695 7696 7697 7698
            if privates:
                c.put(" private(%s)" % ", ".join(privates))
            if firstprivates:
                c.put(" firstprivate(%s)" % ", ".join(firstprivates))
7699 7700 7701 7702 7703 7704 7705 7706

            if self.breaking_label_used:
                shared_vars = [Naming.parallel_why]
                if self.error_label_used:
                    shared_vars.extend(self.parallel_exc)
                    c.put(" private(%s, %s, %s)" % self.pos_info)

                c.put(" shared(%s)" % ', '.join(shared_vars))
7707

7708 7709 7710 7711 7712 7713 7714 7715 7716 7717
    def cleanup_temps(self, code):
        # Now clean up any memoryview slice and object temporaries
        if self.is_parallel and not self.is_nested_prange:
            code.putln("/* Clean up any temporaries */")
            for temp, type in self.temps:
                if type.is_memoryviewslice:
                    code.put_xdecref_memoryviewslice(temp, have_gil=False)
                elif type.is_pyobject:
                    code.put_xdecref(temp, type)
                    code.putln("%s = NULL;" % temp)
7718

7719
    def setup_parallel_control_flow_block(self, code):
7720
        """
7721 7722
        Sets up a block that surrounds the parallel block to determine
        how the parallel section was exited. Any kind of return is
7723 7724 7725
        trapped (break, continue, return, exceptions). This is the idea:

        {
7726
            int why = 0;
7727 7728 7729 7730 7731 7732 7733

            #pragma omp parallel
            {
                return # -> goto new_return_label;
                goto end_parallel;

            new_return_label:
7734
                why = 3;
7735
                goto end_parallel;
7736

7737
            end_parallel:;
7738
                #pragma omp flush(why) # we need to flush for every iteration
7739 7740
            }

7741
            if (why == 3)
7742 7743 7744 7745 7746 7747 7748 7749
                goto old_return_label;
        }
        """
        self.old_loop_labels = code.new_loop_labels()
        self.old_error_label = code.new_error_label()
        self.old_return_label = code.return_label
        code.return_label = code.new_label(name="return")

7750 7751
        code.begin_block() # parallel control flow block
        self.begin_of_parallel_control_block_point = code.insertion_point()
Mark Florisson's avatar
Mark Florisson committed
7752
        self.begin_of_parallel_control_block_point_after_decls = code.insertion_point()
7753

7754 7755
        self.undef_builtin_expect_apple_gcc_bug(code)

7756 7757 7758 7759 7760 7761 7762 7763 7764 7765 7766 7767
    def begin_parallel_block(self, code):
        """
        Each OpenMP thread in a parallel section that contains a with gil block
        must have the thread-state initialized. The call to
        PyGILState_Release() then deallocates our threadstate. If we wouldn't
        do this, each with gil block would allocate and deallocate one, thereby
        losing exception information before it can be saved before leaving the
        parallel section.
        """
        self.begin_of_parallel_block = code.insertion_point()

    def end_parallel_block(self, code):
7768 7769 7770 7771 7772 7773 7774 7775 7776
        """
        To ensure all OpenMP threads have thread states, we ensure the GIL
        in each thread (which creates a thread state if it doesn't exist),
        after which we release the GIL.
        On exit, reacquire the GIL and release the thread state.

        If compiled without OpenMP support (at the C level), then we still have
        to acquire the GIL to decref any object temporaries.
        """
7777 7778 7779 7780
        if self.error_label_used:
            begin_code = self.begin_of_parallel_block
            end_code = code

7781
            begin_code.putln("#ifdef _OPENMP")
7782 7783
            begin_code.put_ensure_gil(declare_gilstate=True)
            begin_code.putln("Py_BEGIN_ALLOW_THREADS")
7784
            begin_code.putln("#endif /* _OPENMP */")
7785

7786
            end_code.putln("#ifdef _OPENMP")
7787
            end_code.putln("Py_END_ALLOW_THREADS")
7788 7789 7790 7791 7792
            end_code.putln("#else")
            end_code.put_safe("{\n")
            end_code.put_ensure_gil()
            end_code.putln("#endif /* _OPENMP */")
            self.cleanup_temps(end_code)
7793
            end_code.put_release_ensured_gil()
7794 7795
            end_code.putln("#ifndef _OPENMP")
            end_code.put_safe("}\n")
7796
            end_code.putln("#endif /* _OPENMP */")
7797

7798 7799 7800 7801
    def trap_parallel_exit(self, code, should_flush=False):
        """
        Trap any kind of return inside a parallel construct. 'should_flush'
        indicates whether the variable should be flushed, which is needed by
7802 7803 7804 7805 7806 7807 7808 7809 7810
        prange to skip the loop. It also indicates whether we need to register
        a continue (we need this for parallel blocks, but not for prange
        loops, as it is a direct jump there).

        It uses the same mechanism as try/finally:
            1 continue
            2 break
            3 return
            4 error
7811
        """
7812
        save_lastprivates_label = code.new_label()
7813
        dont_return_label = code.new_label()
7814 7815 7816

        self.any_label_used = False
        self.breaking_label_used = False
7817
        self.error_label_used = False
7818

7819 7820 7821 7822 7823 7824
        self.parallel_private_temps = []

        all_labels = code.get_all_labels()

        # Figure this out before starting to generate any code
        for label in all_labels:
7825 7826
            if code.label_used(label):
                self.breaking_label_used = (self.breaking_label_used or
7827 7828 7829 7830 7831 7832 7833 7834 7835 7836 7837
                                            label != code.continue_label)
                self.any_label_used = True

        if self.any_label_used:
            code.put_goto(dont_return_label)

        for i, label in enumerate(all_labels):
            if not code.label_used(label):
                continue

            is_continue_label = label == code.continue_label
7838

7839
            code.put_label(label)
7840

7841 7842 7843 7844
            if not (should_flush and is_continue_label):
                if label == code.error_label:
                    self.error_label_used = True
                    self.fetch_parallel_exception(code)
7845

7846
                code.putln("%s = %d;" % (Naming.parallel_why, i + 1))
7847

7848 7849 7850 7851
            if (self.breaking_label_used and self.is_prange and not
                    is_continue_label):
                code.put_goto(save_lastprivates_label)
            else:
7852 7853
                code.put_goto(dont_return_label)

7854
        if self.any_label_used:
7855 7856 7857 7858 7859
            if self.is_prange and self.breaking_label_used:
                # Don't rely on lastprivate, save our lastprivates
                code.put_label(save_lastprivates_label)
                self.save_parallel_vars(code)

7860 7861 7862
            code.put_label(dont_return_label)

            if should_flush and self.breaking_label_used:
7863 7864
                code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_why)

7865 7866 7867 7868 7869 7870 7871 7872 7873 7874 7875 7876 7877 7878 7879 7880 7881 7882 7883 7884 7885 7886 7887
    def save_parallel_vars(self, code):
        """
        The following shenanigans are instated when we break, return or
        propagate errors from a prange. In this case we cannot rely on
        lastprivate() to do its job, as no iterations may have executed yet
        in the last thread, leaving the values undefined. It is most likely
        that the breaking thread has well-defined values of the lastprivate
        variables, so we keep those values.
        """
        section_name = ("__pyx_parallel_lastprivates%d" %
                                            self.critical_section_counter)
        code.putln_openmp("#pragma omp critical(%s)" % section_name)
        ParallelStatNode.critical_section_counter += 1

        code.begin_block() # begin critical section

        c = self.begin_of_parallel_control_block_point

        temp_count = 0
        for entry, (op, lastprivate) in self.privates.iteritems():
            if not lastprivate or entry.type.is_pyobject:
                continue

7888
            type_decl = entry.type.empty_declaration_code()
7889 7890 7891 7892 7893
            temp_cname = "__pyx_parallel_temp%d" % temp_count
            private_cname = entry.cname

            temp_count += 1

7894 7895 7896 7897 7898
            invalid_value = entry.type.invalid_value()
            if invalid_value:
                init = ' = ' + invalid_value
            else:
                init = ''
7899
            # Declare the parallel private in the outer block
7900
            c.putln("%s %s%s;" % (type_decl, temp_cname, init))
7901 7902 7903 7904 7905 7906 7907 7908

            # Initialize before escaping
            code.putln("%s = %s;" % (temp_cname, private_cname))

            self.parallel_private_temps.append((temp_cname, private_cname))

        code.end_block() # end critical section

7909 7910 7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928 7929 7930 7931 7932 7933 7934 7935 7936 7937 7938 7939 7940 7941 7942 7943 7944
    def fetch_parallel_exception(self, code):
        """
        As each OpenMP thread may raise an exception, we need to fetch that
        exception from the threadstate and save it for after the parallel
        section where it can be re-raised in the master thread.

        Although it would seem that __pyx_filename, __pyx_lineno and
        __pyx_clineno are only assigned to under exception conditions (i.e.,
        when we have the GIL), and thus should be allowed to be shared without
        any race condition, they are in fact subject to the same race
        conditions that they were previously when they were global variables
        and functions were allowed to release the GIL:

            thread A                thread B
                acquire
                set lineno
                release
                                        acquire
                                        set lineno
                                        release
                acquire
                fetch exception
                release
                                        skip the fetch

                deallocate threadstate  deallocate threadstate
        """
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

        code.putln_openmp("#pragma omp flush(%s)" % Naming.parallel_exc_type)
        code.putln(
            "if (!%s) {" % Naming.parallel_exc_type)

        code.putln("__Pyx_ErrFetch(&%s, &%s, &%s);" % self.parallel_exc)
        pos_info = chain(*zip(self.parallel_pos_info, self.pos_info))
7945
        code.funcstate.uses_error_indicator = True
7946
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))
7947
        code.put_gotref(Naming.parallel_exc_type)
7948 7949 7950 7951 7952 7953 7954 7955 7956 7957 7958 7959

        code.putln(
            "}")

        code.put_release_ensured_gil()
        code.end_block()

    def restore_parallel_exception(self, code):
        "Re-raise a parallel exception"
        code.begin_block()
        code.put_ensure_gil(declare_gilstate=True)

7960
        code.put_giveref(Naming.parallel_exc_type)
7961 7962 7963 7964 7965 7966
        code.putln("__Pyx_ErrRestore(%s, %s, %s);" % self.parallel_exc)
        pos_info = chain(*zip(self.pos_info, self.parallel_pos_info))
        code.putln("%s = %s; %s = %s; %s = %s;" % tuple(pos_info))

        code.put_release_ensured_gil()
        code.end_block()
7967 7968

    def restore_labels(self, code):
7969 7970 7971 7972 7973 7974
        """
        Restore all old labels. Call this before the 'else' clause to for
        loops and always before ending the parallel control flow block.
        """
        code.set_all_labels(self.old_loop_labels + (self.old_return_label,
                                                    self.old_error_label))
7975

7976 7977 7978 7979 7980 7981 7982
    def end_parallel_control_flow_block(self, code,
                                        break_=False, continue_=False):
        """
        This ends the parallel control flow block and based on how the parallel
        section was exited, takes the corresponding action. The break_ and
        continue_ parameters indicate whether these should be propagated
        outwards:
7983

7984 7985 7986 7987 7988 7989 7990
            for i in prange(...):
                with cython.parallel.parallel():
                    continue

        Here break should be trapped in the parallel block, and propagated to
        the for loop.
        """
7991 7992 7993 7994
        c = self.begin_of_parallel_control_block_point

        # Firstly, always prefer errors over returning, continue or break
        if self.error_label_used:
Mark Florisson's avatar
Mark Florisson committed
7995 7996
            c.putln("const char *%s = NULL; int %s = 0, %s = 0;" %
                                                self.parallel_pos_info)
7997

7998
            c.putln("PyObject *%s = NULL, *%s = NULL, *%s = NULL;" %
Mark Florisson's avatar
Mark Florisson committed
7999
                                                        self.parallel_exc)
8000 8001 8002 8003 8004 8005 8006 8007 8008

            code.putln(
                "if (%s) {" % Naming.parallel_exc_type)
            code.putln("/* This may have been overridden by a continue, "
                       "break or return in another thread. Prefer the error. */")
            code.putln("%s = 4;" % Naming.parallel_why)
            code.putln(
                "}")

8009 8010 8011 8012 8013 8014 8015
        if continue_:
            any_label_used = self.any_label_used
        else:
            any_label_used = self.breaking_label_used

        if any_label_used:
            # __pyx_parallel_why is used, declare and initialize
8016 8017
            c.putln("int %s;" % Naming.parallel_why)
            c.putln("%s = 0;" % Naming.parallel_why)
8018

8019 8020 8021 8022 8023 8024
            code.putln(
                "if (%s) {" % Naming.parallel_why)

            for temp_cname, private_cname in self.parallel_private_temps:
                code.putln("%s = %s;" % (private_cname, temp_cname))

8025
            code.putln("switch (%s) {" % Naming.parallel_why)
8026 8027 8028 8029 8030 8031 8032 8033 8034 8035
            if continue_:
                code.put("    case 1: ")
                code.put_goto(code.continue_label)

            if break_:
                code.put("    case 2: ")
                code.put_goto(code.break_label)

            code.put("    case 3: ")
            code.put_goto(code.return_label)
8036 8037

            if self.error_label_used:
8038
                code.globalstate.use_utility_code(restore_exception_utility_code)
8039 8040 8041
                code.putln("    case 4:")
                self.restore_parallel_exception(code)
                code.put_goto(code.error_label)
8042

8043 8044 8045
            code.putln("}") # end switch
            code.putln(
                "}") # end if
8046 8047

        code.end_block() # end parallel control flow block
8048 8049 8050 8051 8052 8053 8054 8055 8056 8057 8058 8059
        self.redef_builtin_expect_apple_gcc_bug(code)

    # FIXME: improve with version number for OS X Lion
    buggy_platform_macro_condition = "(defined(__APPLE__) || defined(__OSX__))"
    have_expect_condition = "(defined(__GNUC__) && " \
                             "(__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95))))"
    redef_condition = "(%s && %s)" % (buggy_platform_macro_condition, have_expect_condition)

    def undef_builtin_expect_apple_gcc_bug(self, code):
        """
        A bug on OS X Lion disallows __builtin_expect macros. This code avoids them
        """
8060
        if not self.parent:
8061
            code.undef_builtin_expect(self.redef_condition)
8062 8063

    def redef_builtin_expect_apple_gcc_bug(self, code):
8064
        if not self.parent:
8065
            code.redef_builtin_expect(self.redef_condition)
8066

Mark Florisson's avatar
Mark Florisson committed
8067 8068 8069

class ParallelWithBlockNode(ParallelStatNode):
    """
8070
    This node represents a 'with cython.parallel.parallel():' block
Mark Florisson's avatar
Mark Florisson committed
8071 8072
    """

8073 8074 8075 8076 8077 8078 8079 8080 8081
    valid_keyword_arguments = ['num_threads']

    num_threads = None

    def analyse_declarations(self, env):
        super(ParallelWithBlockNode, self).analyse_declarations(env)
        if self.args:
            error(self.pos, "cython.parallel.parallel() does not take "
                            "positional arguments")
8082

Mark Florisson's avatar
Mark Florisson committed
8083 8084
    def generate_execution_code(self, code):
        self.declare_closure_privates(code)
8085
        self.setup_parallel_control_flow_block(code)
Mark Florisson's avatar
Mark Florisson committed
8086 8087 8088

        code.putln("#ifdef _OPENMP")
        code.put("#pragma omp parallel ")
8089 8090

        if self.privates:
8091 8092
            privates = [e.cname for e in self.privates
                                    if not e.type.is_pyobject]
8093
            code.put('private(%s)' % ', '.join(privates))
8094

8095
        self.privatization_insertion_point = code.insertion_point()
8096
        self.put_num_threads(code)
8097
        code.putln("")
8098

8099 8100
        code.putln("#endif /* _OPENMP */")

8101
        code.begin_block() # parallel block
8102
        self.begin_parallel_block(code)
8103
        self.initialize_privates_to_nan(code)
8104
        code.funcstate.start_collecting_temps()
Mark Florisson's avatar
Mark Florisson committed
8105
        self.body.generate_execution_code(code)
8106
        self.trap_parallel_exit(code)
8107
        self.privatize_temps(code)
8108 8109
        self.end_parallel_block(code)
        code.end_block() # end parallel block
8110

8111 8112
        continue_ = code.label_used(code.continue_label)
        break_ = code.label_used(code.break_label)
Mark Florisson's avatar
Mark Florisson committed
8113

8114 8115 8116
        self.restore_labels(code)
        self.end_parallel_control_flow_block(code, break_=break_,
                                             continue_=continue_)
Mark Florisson's avatar
Mark Florisson committed
8117 8118 8119 8120 8121 8122 8123 8124 8125 8126 8127
        self.release_closure_privates(code)


class ParallelRangeNode(ParallelStatNode):
    """
    This node represents a 'for i in cython.parallel.prange():' construct.

    target       NameNode       the target iteration variable
    else_clause  Node or None   the else clause of this loop
    """

8128 8129
    child_attrs = ['body', 'target', 'else_clause', 'args', 'num_threads',
                   'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8130 8131 8132 8133 8134 8135

    body = target = else_clause = args = None

    start = stop = step = None

    is_prange = True
8136

8137
    nogil = None
8138 8139
    schedule = None

8140
    valid_keyword_arguments = ['schedule', 'nogil', 'num_threads', 'chunksize']
Mark Florisson's avatar
Mark Florisson committed
8141

8142 8143 8144 8145 8146
    def __init__(self, pos, **kwds):
        super(ParallelRangeNode, self).__init__(pos, **kwds)
        # Pretend to be a ForInStatNode for control flow analysis
        self.iterator = PassStatNode(pos)

Mark Florisson's avatar
Mark Florisson committed
8147 8148 8149 8150 8151 8152 8153 8154 8155 8156 8157 8158 8159 8160 8161 8162 8163
    def analyse_declarations(self, env):
        super(ParallelRangeNode, self).analyse_declarations(env)
        self.target.analyse_target_declaration(env)
        if self.else_clause is not None:
            self.else_clause.analyse_declarations(env)

        if not self.args or len(self.args) > 3:
            error(self.pos, "Invalid number of positional arguments to prange")
            return

        if len(self.args) == 1:
            self.stop, = self.args
        elif len(self.args) == 2:
            self.start, self.stop = self.args
        else:
            self.start, self.stop, self.step = self.args

Mark Florisson's avatar
Mark Florisson committed
8164 8165 8166
        if hasattr(self.schedule, 'decode'):
            self.schedule = self.schedule.decode('ascii')

Mark Florisson's avatar
Mark Florisson committed
8167 8168
        if self.schedule not in (None, 'static', 'dynamic', 'guided',
                                 'runtime'):
Mark Florisson's avatar
Mark Florisson committed
8169
            error(self.pos, "Invalid schedule argument to prange: %s" %
Mark Florisson's avatar
Mark Florisson committed
8170 8171 8172
                                                        (self.schedule,))

    def analyse_expressions(self, env):
Stefan Behnel's avatar
Stefan Behnel committed
8173
        was_nogil = env.nogil
8174 8175 8176
        if self.nogil:
            env.nogil = True

8177 8178
        if self.target is None:
            error(self.pos, "prange() can only be used as part of a for loop")
8179
            return self
Mark Florisson's avatar
Mark Florisson committed
8180

8181
        self.target = self.target.analyse_target_types(env)
Mark Florisson's avatar
Mark Florisson committed
8182

8183 8184 8185 8186 8187 8188 8189 8190
        if not self.target.type.is_numeric:
            # Not a valid type, assume one for now anyway

            if not self.target.type.is_pyobject:
                # nogil_check will catch the is_pyobject case
                error(self.target.pos,
                      "Must be of numeric type, not %s" % self.target.type)

8191
            self.index_type = PyrexTypes.c_py_ssize_t_type
8192 8193
        else:
            self.index_type = self.target.type
8194 8195 8196 8197
            if not self.index_type.signed:
                warning(self.target.pos,
                        "Unsigned index type not allowed before OpenMP 3.0",
                        level=2)
Mark Florisson's avatar
Mark Florisson committed
8198 8199 8200 8201 8202 8203 8204 8205 8206

        # Setup start, stop and step, allocating temps if needed
        self.names = 'start', 'stop', 'step'
        start_stop_step = self.start, self.stop, self.step

        for node, name in zip(start_stop_step, self.names):
            if node is not None:
                node.analyse_types(env)
                if not node.type.is_numeric:
8207 8208
                    error(node.pos, "%s argument must be numeric" % name)
                    continue
Mark Florisson's avatar
Mark Florisson committed
8209 8210 8211 8212 8213 8214 8215 8216 8217 8218 8219

                if not node.is_literal:
                    node = node.coerce_to_temp(env)
                    setattr(self, name, node)

                # As we range from 0 to nsteps, computing the index along the
                # way, we need a fitting type for 'i' and 'nsteps'
                self.index_type = PyrexTypes.widest_numeric_type(
                                        self.index_type, node.type)

        if self.else_clause is not None:
8220
            self.else_clause = self.else_clause.analyse_expressions(env)
Mark Florisson's avatar
Mark Florisson committed
8221

8222 8223 8224 8225 8226 8227 8228 8229
        # Although not actually an assignment in this scope, it should be
        # treated as such to ensure it is unpacked if a closure temp, and to
        # ensure lastprivate behaviour and propagation. If the target index is
        # not a NameNode, it won't have an entry, and an error was issued by
        # ParallelRangeTransform
        if hasattr(self.target, 'entry'):
            self.assignments[self.target.entry] = self.target.pos, None

8230
        node = super(ParallelRangeNode, self).analyse_expressions(env)
8231

8232 8233 8234
        if node.chunksize:
            if not node.schedule:
                error(node.chunksize.pos,
8235
                      "Must provide schedule with chunksize")
8236 8237
            elif node.schedule == 'runtime':
                error(node.chunksize.pos,
8238
                      "Chunksize not valid for the schedule runtime")
8239 8240 8241 8242
            elif (node.chunksize.type.is_int and
                  node.chunksize.is_literal and
                  node.chunksize.compile_time_value(env) <= 0):
                error(node.chunksize.pos, "Chunksize must not be negative")
8243

8244 8245
            node.chunksize = node.chunksize.coerce_to(
                PyrexTypes.c_int_type, env).coerce_to_temp(env)
8246

8247
        if node.nogil:
8248 8249
            env.nogil = was_nogil

8250 8251 8252
        node.is_nested_prange = node.parent and node.parent.is_prange
        if node.is_nested_prange:
            parent = node
8253 8254 8255
            while parent.parent and parent.parent.is_prange:
                parent = parent.parent

8256 8257 8258 8259
            parent.assignments.update(node.assignments)
            parent.privates.update(node.privates)
            parent.assigned_nodes.extend(node.assigned_nodes)
        return node
8260

Mark Florisson's avatar
Mark Florisson committed
8261 8262
    def nogil_check(self, env):
        names = 'start', 'stop', 'step', 'target'
8263
        nodes = self.start, self.stop, self.step, self.target
Mark Florisson's avatar
Mark Florisson committed
8264 8265 8266 8267 8268 8269 8270 8271 8272 8273 8274 8275 8276 8277 8278 8279 8280 8281 8282 8283 8284 8285 8286 8287 8288 8289 8290 8291 8292 8293 8294 8295 8296 8297 8298 8299 8300 8301 8302 8303 8304 8305 8306 8307 8308
        for name, node in zip(names, nodes):
            if node is not None and node.type.is_pyobject:
                error(node.pos, "%s may not be a Python object "
                                "as we don't have the GIL" % name)

    def generate_execution_code(self, code):
        """
        Generate code in the following steps

            1)  copy any closure variables determined thread-private
                into temporaries

            2)  allocate temps for start, stop and step

            3)  generate a loop that calculates the total number of steps,
                which then computes the target iteration variable for every step:

                    for i in prange(start, stop, step):
                        ...

                becomes

                    nsteps = (stop - start) / step;
                    i = start;

                    #pragma omp parallel for lastprivate(i)
                    for (temp = 0; temp < nsteps; temp++) {
                        i = start + step * temp;
                        ...
                    }

                Note that accumulation of 'i' would have a data dependency
                between iterations.

                Also, you can't do this

                    for (i = start; i < stop; i += step)
                        ...

                as the '<' operator should become '>' for descending loops.
                'for i from x < i < y:' does not suffer from this problem
                as the relational operator is known at compile time!

            4) release our temps and write back any private closure variables
        """
8309
        self.declare_closure_privates(code)
Mark Florisson's avatar
Mark Florisson committed
8310 8311 8312 8313 8314 8315 8316 8317 8318 8319 8320 8321 8322 8323 8324 8325 8326 8327 8328 8329 8330 8331 8332 8333 8334 8335 8336 8337 8338 8339 8340

        # This can only be a NameNode
        target_index_cname = self.target.entry.cname

        # This will be used as the dict to format our code strings, holding
        # the start, stop , step, temps and target cnames
        fmt_dict = {
            'target': target_index_cname,
        }

        # Setup start, stop and step, allocating temps if needed
        start_stop_step = self.start, self.stop, self.step
        defaults = '0', '0', '1'
        for node, name, default in zip(start_stop_step, self.names, defaults):
            if node is None:
                result = default
            elif node.is_literal:
                result = node.get_constant_c_result_code()
            else:
                node.generate_evaluation_code(code)
                result = node.result()

            fmt_dict[name] = result

        fmt_dict['i'] = code.funcstate.allocate_temp(self.index_type, False)
        fmt_dict['nsteps'] = code.funcstate.allocate_temp(self.index_type, False)

        # TODO: check if the step is 0 and if so, raise an exception in a
        # 'with gil' block. For now, just abort
        code.putln("if (%(step)s == 0) abort();" % fmt_dict)

8341
        self.setup_parallel_control_flow_block(code) # parallel control flow block
8342

8343
        self.control_flow_var_code_point = code.insertion_point()
8344

8345
        # Note: nsteps is private in an outer scope if present
Mark Florisson's avatar
Mark Florisson committed
8346 8347
        code.putln("%(nsteps)s = (%(stop)s - %(start)s) / %(step)s;" % fmt_dict)

8348 8349 8350 8351 8352 8353 8354
        # The target iteration variable might not be initialized, do it only if
        # we are executing at least 1 iteration, otherwise we should leave the
        # target unaffected. The target iteration variable is firstprivate to
        # shut up compiler warnings caused by lastprivate, as the compiler
        # erroneously believes that nsteps may be <= 0, leaving the private
        # target index uninitialized
        code.putln("if (%(nsteps)s > 0)" % fmt_dict)
8355
        code.begin_block() # if block
Mark Florisson's avatar
Mark Florisson committed
8356
        self.generate_loop(code, fmt_dict)
8357
        code.end_block() # end if block
Mark Florisson's avatar
Mark Florisson committed
8358

8359 8360 8361
        self.restore_labels(code)

        if self.else_clause:
8362
            if self.breaking_label_used:
8363
                code.put("if (%s < 2)" % Naming.parallel_why)
8364 8365 8366 8367 8368 8369 8370

            code.begin_block() # else block
            code.putln("/* else */")
            self.else_clause.generate_execution_code(code)
            code.end_block() # end else block

        # ------ cleanup ------
8371
        self.end_parallel_control_flow_block(code) # end parallel control flow block
8372

Mark Florisson's avatar
Mark Florisson committed
8373 8374
        # And finally, release our privates and write back any closure
        # variables
8375
        for temp in start_stop_step + (self.chunksize, self.num_threads):
Mark Florisson's avatar
Mark Florisson committed
8376 8377 8378 8379 8380 8381 8382 8383 8384 8385
            if temp is not None:
                temp.generate_disposal_code(code)
                temp.free_temps(code)

        code.funcstate.release_temp(fmt_dict['i'])
        code.funcstate.release_temp(fmt_dict['nsteps'])

        self.release_closure_privates(code)

    def generate_loop(self, code, fmt_dict):
8386 8387 8388 8389
        if self.is_nested_prange:
            code.putln("#if 0")
        else:
            code.putln("#ifdef _OPENMP")
Mark Florisson's avatar
Mark Florisson committed
8390 8391 8392

        if not self.is_parallel:
            code.put("#pragma omp for")
8393
            self.privatization_insertion_point = code.insertion_point()
8394
            reduction_codepoint = self.parent.privatization_insertion_point
Mark Florisson's avatar
Mark Florisson committed
8395
        else:
8396 8397
            code.put("#pragma omp parallel")
            self.privatization_insertion_point = code.insertion_point()
8398
            reduction_codepoint = self.privatization_insertion_point
8399 8400 8401 8402 8403 8404 8405 8406
            code.putln("")
            code.putln("#endif /* _OPENMP */")

            code.begin_block() # pragma omp parallel begin block

            # Initialize the GIL if needed for this thread
            self.begin_parallel_block(code)

8407 8408 8409 8410
            if self.is_nested_prange:
                code.putln("#if 0")
            else:
                code.putln("#ifdef _OPENMP")
8411
            code.put("#pragma omp for")
Mark Florisson's avatar
Mark Florisson committed
8412

8413
        for entry, (op, lastprivate) in self.privates.iteritems():
Mark Florisson's avatar
Mark Florisson committed
8414
            # Don't declare the index variable as a reduction
8415
            if op and op in "+*-&^|" and entry != self.target.entry:
8416 8417 8418
                if entry.type.is_pyobject:
                    error(self.pos, "Python objects cannot be reductions")
                else:
8419 8420 8421 8422
                    #code.put(" reduction(%s:%s)" % (op, entry.cname))
                    # This is the only way reductions + nesting works in gcc4.5
                    reduction_codepoint.put(
                                " reduction(%s:%s)" % (op, entry.cname))
8423
            else:
8424 8425
                if entry == self.target.entry:
                    code.put(" firstprivate(%s)" % entry.cname)
8426 8427
                    code.put(" lastprivate(%s)" % entry.cname)
                    continue
Mark Florisson's avatar
Mark Florisson committed
8428

8429
                if not entry.type.is_pyobject:
8430 8431 8432 8433
                    if lastprivate:
                        private = 'lastprivate'
                    else:
                        private = 'private'
Mark Florisson's avatar
Mark Florisson committed
8434

8435
                    code.put(" %s(%s)" % (private, entry.cname))
8436

Mark Florisson's avatar
Mark Florisson committed
8437
        if self.schedule:
8438 8439 8440 8441 8442 8443 8444
            if self.chunksize:
                chunksize = ", %s" % self.evaluate_before_block(code,
                                                                self.chunksize)
            else:
                chunksize = ""

            code.put(" schedule(%s%s)" % (self.schedule, chunksize))
8445

8446
        self.put_num_threads(reduction_codepoint)
8447

8448 8449
        code.putln("")
        code.putln("#endif /* _OPENMP */")
Mark Florisson's avatar
Mark Florisson committed
8450 8451

        code.put("for (%(i)s = 0; %(i)s < %(nsteps)s; %(i)s++)" % fmt_dict)
8452
        code.begin_block() # for loop block
8453

8454
        guard_around_body_codepoint = code.insertion_point()
8455

8456 8457
        # Start if guard block around the body. This may be unnecessary, but
        # at least it doesn't spoil indentation
Mark Florisson's avatar
Mark Florisson committed
8458
        code.begin_block()
8459

8460
        code.putln("%(target)s = %(start)s + %(step)s * %(i)s;" % fmt_dict)
8461 8462
        self.initialize_privates_to_nan(code, exclude=self.target.entry)

8463 8464 8465
        if self.is_parallel:
            code.funcstate.start_collecting_temps()

Mark Florisson's avatar
Mark Florisson committed
8466
        self.body.generate_execution_code(code)
8467
        self.trap_parallel_exit(code, should_flush=True)
8468 8469
        self.privatize_temps(code)

8470 8471 8472
        if self.breaking_label_used:
            # Put a guard around the loop body in case return, break or
            # exceptions might be used
8473
            guard_around_body_codepoint.putln("if (%s < 2)" % Naming.parallel_why)
8474

8475
        code.end_block() # end guard around loop body
8476
        code.end_block() # end for loop block
Mark Florisson's avatar
Mark Florisson committed
8477

8478 8479 8480 8481 8482
        if self.is_parallel:
            # Release the GIL and deallocate the thread state
            self.end_parallel_block(code)
            code.end_block() # pragma omp parallel end block

8483

8484 8485 8486 8487 8488 8489 8490 8491
class CnameDecoratorNode(StatNode):
    """
    This node is for the cname decorator in CythonUtilityCode:

        @cname('the_cname')
        cdef func(...):
            ...

8492 8493
    In case of a cdef class the cname specifies the objstruct_cname.

8494 8495 8496 8497 8498 8499 8500 8501
    node        the node to which the cname decorator is applied
    cname       the cname the node should get
    """

    child_attrs = ['node']

    def analyse_declarations(self, env):
        self.node.analyse_declarations(env)
8502

8503 8504 8505 8506 8507 8508
        node = self.node
        if isinstance(node, CompilerDirectivesNode):
            node = node.body.stats[0]

        self.is_function = isinstance(node, FuncDefNode)
        is_struct_or_enum = isinstance(node, (CStructOrUnionDefNode,
8509
                                                   CEnumDefNode))
8510
        e = node.entry
8511 8512 8513 8514

        if self.is_function:
            e.cname = self.cname
            e.func_cname = self.cname
8515
            e.used = True
8516 8517
            if e.pyfunc_cname and '.' in e.pyfunc_cname:
                e.pyfunc_cname = self.mangle(e.pyfunc_cname)
8518 8519
        elif is_struct_or_enum:
            e.cname = e.type.cname = self.cname
8520
        else:
8521
            scope = node.scope
8522 8523

            e.cname = self.cname
8524
            e.type.objstruct_cname = self.cname + '_obj'
8525
            e.type.typeobj_cname = Naming.typeobj_prefix + self.cname
8526
            e.type.typeptr_cname = self.cname + '_type'
8527
            e.type.scope.namespace_cname = e.type.typeptr_cname
8528 8529 8530 8531 8532 8533 8534

            e.as_variable.cname = py_object_type.cast_code(e.type.typeptr_cname)

            scope.scope_prefix = self.cname + "_"

            for name, entry in scope.entries.iteritems():
                if entry.func_cname:
8535 8536 8537 8538 8539 8540 8541 8542 8543
                    entry.func_cname = self.mangle(entry.cname)
                if entry.pyfunc_cname:
                    entry.pyfunc_cname = self.mangle(entry.pyfunc_cname)

    def mangle(self, cname):
        if '.' in cname:
            # remove __pyx_base from func_cname
            cname = cname.split('.')[-1]
        return '%s_%s' % (self.cname, cname)
8544 8545

    def analyse_expressions(self, env):
8546 8547
        self.node = self.node.analyse_expressions(env)
        return self
8548 8549

    def generate_function_definitions(self, env, code):
8550
        "Ensure a prototype for every @cname method in the right place"
8551 8552 8553 8554 8555 8556 8557 8558
        if self.is_function and env.is_c_class_scope:
            # method in cdef class, generate a prototype in the header
            h_code = code.globalstate['utility_code_proto']

            if isinstance(self.node, DefNode):
                self.node.generate_function_header(
                            h_code, with_pymethdef=False, proto_only=True)
            else:
8559
                from . import ModuleNode
8560 8561 8562 8563 8564 8565 8566 8567 8568 8569 8570 8571 8572 8573 8574 8575
                entry = self.node.entry
                cname = entry.cname
                entry.cname = entry.func_cname

                ModuleNode.generate_cfunction_declaration(
                        entry,
                        env.global_scope(),
                        h_code,
                        definition=True)

                entry.cname = cname

        self.node.generate_function_definitions(env, code)

    def generate_execution_code(self, code):
        self.node.generate_execution_code(code)
Mark Florisson's avatar
Mark Florisson committed
8576

8577

William Stein's avatar
William Stein committed
8578 8579 8580 8581 8582 8583
#------------------------------------------------------------------------------------
#
#  Runtime support code
#
#------------------------------------------------------------------------------------

Robert Bradshaw's avatar
Robert Bradshaw committed
8584
if Options.gcc_branch_hints:
Stefan Behnel's avatar
Stefan Behnel committed
8585
    branch_prediction_macros = """
8586 8587 8588 8589 8590 8591
/* Test for GCC > 2.95 */
#if defined(__GNUC__) \
    && (__GNUC__ > 2 || (__GNUC__ == 2 && (__GNUC_MINOR__ > 95)))
  #define likely(x)   __builtin_expect(!!(x), 1)
  #define unlikely(x) __builtin_expect(!!(x), 0)
#else /* !__GNUC__ or GCC < 2.95 */
8592 8593
  #define likely(x)   (x)
  #define unlikely(x) (x)
8594
#endif /* __GNUC__ */
Stefan Behnel's avatar
Stefan Behnel committed
8595
"""
Robert Bradshaw's avatar
Robert Bradshaw committed
8596
else:
Stefan Behnel's avatar
Stefan Behnel committed
8597
    branch_prediction_macros = """
Robert Bradshaw's avatar
Robert Bradshaw committed
8598 8599
#define likely(x)   (x)
#define unlikely(x) (x)
Stefan Behnel's avatar
Stefan Behnel committed
8600
"""
William Stein's avatar
William Stein committed
8601 8602 8603

#------------------------------------------------------------------------------------

8604 8605
printing_utility_code = UtilityCode.load_cached("Print", "Printing.c")
printing_one_utility_code = UtilityCode.load_cached("PrintOne", "Printing.c")
8606

William Stein's avatar
William Stein committed
8607 8608
#------------------------------------------------------------------------------------

8609 8610 8611 8612 8613 8614 8615
# Exception raising code
#
# Exceptions are raised by __Pyx_Raise() and stored as plain
# type/value/tb in PyThreadState->curexc_*.  When being caught by an
# 'except' statement, curexc_* is moved over to exc_* by
# __Pyx_GetException()

8616 8617 8618 8619 8620
restore_exception_utility_code = UtilityCode.load_cached("PyErrFetchRestore", "Exceptions.c")
raise_utility_code = UtilityCode.load_cached("RaiseException", "Exceptions.c")
get_exception_utility_code = UtilityCode.load_cached("GetException", "Exceptions.c")
swap_exception_utility_code = UtilityCode.load_cached("SwapException", "Exceptions.c")
reset_exception_utility_code = UtilityCode.load_cached("SaveResetException", "Exceptions.c")
8621
traceback_utility_code = UtilityCode.load_cached("AddTraceback", "Exceptions.c")
8622 8623 8624 8625 8626 8627

#------------------------------------------------------------------------------------

get_exception_tuple_utility_code = UtilityCode(proto="""
static PyObject *__Pyx_GetExceptionTuple(void); /*proto*/
""",
8628 8629 8630
# I doubt that calling __Pyx_GetException() here is correct as it moves
# the exception from tstate->curexc_* to tstate->exc_*, which prevents
# exception handlers later on from receiving it.
8631 8632 8633 8634 8635 8636 8637 8638 8639 8640 8641 8642 8643 8644 8645 8646 8647 8648 8649
impl = """
static PyObject *__Pyx_GetExceptionTuple(void) {
    PyObject *type = NULL, *value = NULL, *tb = NULL;
    if (__Pyx_GetException(&type, &value, &tb) == 0) {
        PyObject* exc_info = PyTuple_New(3);
        if (exc_info) {
            Py_INCREF(type);
            Py_INCREF(value);
            Py_INCREF(tb);
            PyTuple_SET_ITEM(exc_info, 0, type);
            PyTuple_SET_ITEM(exc_info, 1, value);
            PyTuple_SET_ITEM(exc_info, 2, tb);
            return exc_info;
        }
    }
    return NULL;
}
""",
requires=[get_exception_utility_code])