ParseTreeTransforms.py 115 KB
Newer Older
1 2 3 4
from __future__ import absolute_import

import copy

5
import cython
6
cython.declare(PyrexTypes=object, Naming=object, ExprNodes=object, Nodes=object,
7 8
               Options=object, UtilNodes=object, LetNode=object,
               LetRefNode=object, TreeFragment=object, EncodedString=object,
9 10
               error=object, warning=object, copy=object)

11 12 13 14 15 16
from . import PyrexTypes
from . import Naming
from . import ExprNodes
from . import Nodes
from . import Options
from . import Builtin
17

18 19 20 21 22 23 24
from .Visitor import VisitorTransform, TreeVisitor
from .Visitor import CythonTransform, EnvTransform, ScopeTrackingTransform
from .UtilNodes import LetNode, LetRefNode, ResultRefNode
from .TreeFragment import TreeFragment
from .StringEncoding import EncodedString
from .Errors import error, warning, CompileError, InternalError
from .Code import UtilityCode
25

26 27 28 29 30 31 32 33 34 35 36 37

class NameNodeCollector(TreeVisitor):
    """Collect all NameNodes of a (sub-)tree in the ``name_nodes``
    attribute.
    """
    def __init__(self):
        super(NameNodeCollector, self).__init__()
        self.name_nodes = []

    def visit_NameNode(self, node):
        self.name_nodes.append(node)

38 39 40
    def visit_Node(self, node):
        self._visitchildren(node, None)

41

42
class SkipDeclarations(object):
43
    """
44 45 46 47 48
    Variable and function declarations can often have a deep tree structure,
    and yet most transformations don't need to descend to this depth.

    Declaration nodes are removed after AnalyseDeclarationsTransform, so there
    is no need to use this for transformations after that point.
49 50 51
    """
    def visit_CTypeDefNode(self, node):
        return node
52

53 54
    def visit_CVarDefNode(self, node):
        return node
55

56 57
    def visit_CDeclaratorNode(self, node):
        return node
58

59 60
    def visit_CBaseTypeNode(self, node):
        return node
61

62 63 64 65 66 67
    def visit_CEnumDefNode(self, node):
        return node

    def visit_CStructOrUnionDefNode(self, node):
        return node

68
class NormalizeTree(CythonTransform):
69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88
    """
    This transform fixes up a few things after parsing
    in order to make the parse tree more suitable for
    transforms.

    a) After parsing, blocks with only one statement will
    be represented by that statement, not by a StatListNode.
    When doing transforms this is annoying and inconsistent,
    as one cannot in general remove a statement in a consistent
    way and so on. This transform wraps any single statements
    in a StatListNode containing a single statement.

    b) The PassStatNode is a noop and serves no purpose beyond
    plugging such one-statement blocks; i.e., once parsed a
`    "pass" can just as well be represented using an empty
    StatListNode. This means less special cases to worry about
    in subsequent transforms (one always checks to see if a
    StatListNode has no children to see if the block is empty).
    """

89 90
    def __init__(self, context):
        super(NormalizeTree, self).__init__(context)
91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106
        self.is_in_statlist = False
        self.is_in_expr = False

    def visit_ExprNode(self, node):
        stacktmp = self.is_in_expr
        self.is_in_expr = True
        self.visitchildren(node)
        self.is_in_expr = stacktmp
        return node

    def visit_StatNode(self, node, is_listcontainer=False):
        stacktmp = self.is_in_statlist
        self.is_in_statlist = is_listcontainer
        self.visitchildren(node)
        self.is_in_statlist = stacktmp
        if not self.is_in_statlist and not self.is_in_expr:
107
            return Nodes.StatListNode(pos=node.pos, stats=[node])
108 109 110 111 112 113 114 115 116 117 118
        else:
            return node

    def visit_StatListNode(self, node):
        self.is_in_statlist = True
        self.visitchildren(node)
        self.is_in_statlist = False
        return node

    def visit_ParallelAssignmentNode(self, node):
        return self.visit_StatNode(node, True)
119

120 121 122 123 124 125
    def visit_CEnumDefNode(self, node):
        return self.visit_StatNode(node, True)

    def visit_CStructOrUnionDefNode(self, node):
        return self.visit_StatNode(node, True)

126
    def visit_PassStatNode(self, node):
127
        """Eliminate PassStatNode"""
128
        if not self.is_in_statlist:
129
            return Nodes.StatListNode(pos=node.pos, stats=[])
130 131 132
        else:
            return []

133 134 135
    def visit_ExprStatNode(self, node):
        """Eliminate useless string literals"""
        if node.expr.is_string_literal:
Stefan Behnel's avatar
Stefan Behnel committed
136 137 138
            return self.visit_PassStatNode(node)
        else:
            return self.visit_StatNode(node)
139

140
    def visit_CDeclaratorNode(self, node):
141
        return node
142

143

144 145 146
class PostParseError(CompileError): pass

# error strings checked by unit tests, so define them
147
ERR_CDEF_INCLASS = 'Cannot assign default value to fields in cdef classes, structs or unions'
148 149
ERR_BUF_DEFAULTS = 'Invalid buffer defaults specification (see docs)'
ERR_INVALID_SPECIALATTR_TYPE = 'Special attributes must not have a type declared'
150
class PostParse(ScopeTrackingTransform):
151 152 153 154 155 156 157
    """
    Basic interpretation of the parse tree, as well as validity
    checking that can be done on a very basic level on the parse
    tree (while still not being a problem with the basic syntax,
    as such).

    Specifically:
158
    - Default values to cdef assignments are turned into single
159 160
    assignments following the declaration (everywhere but in class
    bodies, where they raise a compile error)
161

162 163
    - Interpret some node structures into Python runtime values.
    Some nodes take compile-time arguments (currently:
164
    TemplatedTypeNode[args] and __cythonbufferdefaults__ = {args}),
165 166 167 168 169 170 171 172
    which should be interpreted. This happens in a general way
    and other steps should be taken to ensure validity.

    Type arguments cannot be interpreted in this way.

    - For __cythonbufferdefaults__ the arguments are checked for
    validity.

Robert Bradshaw's avatar
Robert Bradshaw committed
173
    TemplatedTypeNode has its directives interpreted:
174 175
    Any first positional argument goes into the "dtype" attribute,
    any "ndim" keyword argument goes into the "ndim" attribute and
176
    so on. Also it is checked that the directive combination is valid.
177 178
    - __cythonbufferdefaults__ attributes are parsed and put into the
    type information.
179 180 181 182 183 184

    Note: Currently Parsing.py does a lot of interpretation and
    reorganization that can be refactored into this transform
    if a more pure Abstract Syntax Tree is wanted.
    """

185 186 187 188 189 190
    def __init__(self, context):
        super(PostParse, self).__init__(context)
        self.specialattribute_handlers = {
            '__cythonbufferdefaults__' : self.handle_bufferdefaults
        }

191
    def visit_ModuleNode(self, node):
Stefan Behnel's avatar
Stefan Behnel committed
192
        self.lambda_counter = 1
193
        self.genexpr_counter = 1
194
        return super(PostParse, self).visit_ModuleNode(node)
195

Stefan Behnel's avatar
Stefan Behnel committed
196 197 198 199 200
    def visit_LambdaNode(self, node):
        # unpack a lambda expression into the corresponding DefNode
        lambda_id = self.lambda_counter
        self.lambda_counter += 1
        node.lambda_name = EncodedString(u'lambda%d' % lambda_id)
Vitja Makarov's avatar
Vitja Makarov committed
201 202
        collector = YieldNodeCollector()
        collector.visitchildren(node.result_expr)
203
        if collector.yields or collector.awaits or isinstance(node.result_expr, ExprNodes.YieldExprNode):
Vitja Makarov's avatar
Vitja Makarov committed
204 205
            body = Nodes.ExprStatNode(
                node.result_expr.pos, expr=node.result_expr)
Vitja Makarov's avatar
Vitja Makarov committed
206 207 208
        else:
            body = Nodes.ReturnStatNode(
                node.result_expr.pos, value=node.result_expr)
Stefan Behnel's avatar
Stefan Behnel committed
209 210 211 212
        node.def_node = Nodes.DefNode(
            node.pos, name=node.name, lambda_name=node.lambda_name,
            args=node.args, star_arg=node.star_arg,
            starstar_arg=node.starstar_arg,
Vitja Makarov's avatar
Vitja Makarov committed
213
            body=body, doc=None)
Stefan Behnel's avatar
Stefan Behnel committed
214 215
        self.visitchildren(node)
        return node
216 217 218 219 220 221 222

    def visit_GeneratorExpressionNode(self, node):
        # unpack a generator expression into the corresponding DefNode
        genexpr_id = self.genexpr_counter
        self.genexpr_counter += 1
        node.genexpr_name = EncodedString(u'genexpr%d' % genexpr_id)

Vitja Makarov's avatar
Vitja Makarov committed
223
        node.def_node = Nodes.DefNode(node.pos, name=node.name,
224 225 226 227
                                      doc=None,
                                      args=[], star_arg=None,
                                      starstar_arg=None,
                                      body=node.loop)
Stefan Behnel's avatar
Stefan Behnel committed
228 229 230
        self.visitchildren(node)
        return node

231
    # cdef variables
232
    def handle_bufferdefaults(self, decl):
233
        if not isinstance(decl.default, ExprNodes.DictNode):
234
            raise PostParseError(decl.pos, ERR_BUF_DEFAULTS)
235 236
        self.scope_node.buffer_defaults_node = decl.default
        self.scope_node.buffer_defaults_pos = decl.pos
237

238 239
    def visit_CVarDefNode(self, node):
        # This assumes only plain names and pointers are assignable on
240 241 242
        # declaration. Also, it makes use of the fact that a cdef decl
        # must appear before the first use, so we don't have to deal with
        # "i = 3; cdef int i = i" and can simply move the nodes around.
243 244
        try:
            self.visitchildren(node)
245 246 247 248
            stats = [node]
            newdecls = []
            for decl in node.declarators:
                declbase = decl
249
                while isinstance(declbase, Nodes.CPtrDeclaratorNode):
250
                    declbase = declbase.base
251
                if isinstance(declbase, Nodes.CNameDeclaratorNode):
252
                    if declbase.default is not None:
253
                        if self.scope_type in ('cclass', 'pyclass', 'struct'):
254
                            if isinstance(self.scope_node, Nodes.CClassDefNode):
255 256 257 258 259 260 261
                                handler = self.specialattribute_handlers.get(decl.name)
                                if handler:
                                    if decl is not declbase:
                                        raise PostParseError(decl.pos, ERR_INVALID_SPECIALATTR_TYPE)
                                    handler(decl)
                                    continue # Remove declaration
                            raise PostParseError(decl.pos, ERR_CDEF_INCLASS)
262
                        first_assignment = self.scope_type != 'module'
263 264
                        stats.append(Nodes.SingleAssignmentNode(node.pos,
                            lhs=ExprNodes.NameNode(node.pos, name=declbase.name),
265
                            rhs=declbase.default, first=first_assignment))
266 267 268 269
                        declbase.default = None
                newdecls.append(decl)
            node.declarators = newdecls
            return stats
270 271 272 273 274
        except PostParseError, e:
            # An error in a cdef clause is ok, simply remove the declaration
            # and try to move on to report more errors
            self.context.nonfatal_error(e)
            return None
275

Stefan Behnel's avatar
Stefan Behnel committed
276 277
    # Split parallel assignments (a,b = b,a) into separate partial
    # assignments that are executed rhs-first using temps.  This
Stefan Behnel's avatar
Stefan Behnel committed
278 279 280 281
    # restructuring must be applied before type analysis so that known
    # types on rhs and lhs can be matched directly.  It is required in
    # the case that the types cannot be coerced to a Python type in
    # order to assign from a tuple.
282 283 284 285 286 287 288 289 290 291

    def visit_SingleAssignmentNode(self, node):
        self.visitchildren(node)
        return self._visit_assignment_node(node, [node.lhs, node.rhs])

    def visit_CascadedAssignmentNode(self, node):
        self.visitchildren(node)
        return self._visit_assignment_node(node, node.lhs_list + [node.rhs])

    def _visit_assignment_node(self, node, expr_list):
292 293 294
        """Flatten parallel assignments into separate single
        assignments or cascaded assignments.
        """
295 296
        if sum([ 1 for expr in expr_list
                 if expr.is_sequence_constructor or expr.is_string_literal ]) < 2:
297 298 299
            # no parallel assignments => nothing to do
            return node

300 301
        expr_list_list = []
        flatten_parallel_assignments(expr_list, expr_list_list)
302 303 304
        temp_refs = []
        eliminate_rhs_duplicates(expr_list_list, temp_refs)

305 306 307 308 309
        nodes = []
        for expr_list in expr_list_list:
            lhs_list = expr_list[:-1]
            rhs = expr_list[-1]
            if len(lhs_list) == 1:
310
                node = Nodes.SingleAssignmentNode(rhs.pos,
311 312 313 314 315
                    lhs = lhs_list[0], rhs = rhs)
            else:
                node = Nodes.CascadedAssignmentNode(rhs.pos,
                    lhs_list = lhs_list, rhs = rhs)
            nodes.append(node)
316

317
        if len(nodes) == 1:
318 319 320 321 322 323 324 325 326 327 328 329 330
            assign_node = nodes[0]
        else:
            assign_node = Nodes.ParallelAssignmentNode(nodes[0].pos, stats = nodes)

        if temp_refs:
            duplicates_and_temps = [ (temp.expression, temp)
                                     for temp in temp_refs ]
            sort_common_subsequences(duplicates_and_temps)
            for _, temp_ref in duplicates_and_temps[::-1]:
                assign_node = LetNode(temp_ref, assign_node)

        return assign_node

331 332 333 334 335 336 337 338 339 340 341 342 343
    def _flatten_sequence(self, seq, result):
        for arg in seq.args:
            if arg.is_sequence_constructor:
                self._flatten_sequence(arg, result)
            else:
                result.append(arg)
        return result

    def visit_DelStatNode(self, node):
        self.visitchildren(node)
        node.args = self._flatten_sequence(node, [])
        return node

344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362
    def visit_ExceptClauseNode(self, node):
        if node.is_except_as:
            # except-as must delete NameNode target at the end
            del_target = Nodes.DelStatNode(
                node.pos,
                args=[ExprNodes.NameNode(
                    node.target.pos, name=node.target.name)],
                ignore_nonexisting=True)
            node.body = Nodes.StatListNode(
                node.pos,
                stats=[Nodes.TryFinallyStatNode(
                    node.pos,
                    body=node.body,
                    finally_clause=Nodes.StatListNode(
                        node.pos,
                        stats=[del_target]))])
        self.visitchildren(node)
        return node

363

364 365 366 367 368 369
def eliminate_rhs_duplicates(expr_list_list, ref_node_sequence):
    """Replace rhs items by LetRefNodes if they appear more than once.
    Creates a sequence of LetRefNodes that set up the required temps
    and appends them to ref_node_sequence.  The input list is modified
    in-place.
    """
Robert Bradshaw's avatar
Robert Bradshaw committed
370
    seen_nodes = set()
371 372 373 374 375 376 377 378 379 380 381
    ref_nodes = {}
    def find_duplicates(node):
        if node.is_literal or node.is_name:
            # no need to replace those; can't include attributes here
            # as their access is not necessarily side-effect free
            return
        if node in seen_nodes:
            if node not in ref_nodes:
                ref_node = LetRefNode(node)
                ref_nodes[node] = ref_node
                ref_node_sequence.append(ref_node)
382
        else:
383 384 385 386 387 388 389 390 391 392 393 394 395 396 397
            seen_nodes.add(node)
            if node.is_sequence_constructor:
                for item in node.args:
                    find_duplicates(item)

    for expr_list in expr_list_list:
        rhs = expr_list[-1]
        find_duplicates(rhs)
    if not ref_nodes:
        return

    def substitute_nodes(node):
        if node in ref_nodes:
            return ref_nodes[node]
        elif node.is_sequence_constructor:
398
            node.args = list(map(substitute_nodes, node.args))
399
        return node
400

401 402 403
    # replace nodes inside of the common subexpressions
    for node in ref_nodes:
        if node.is_sequence_constructor:
404
            node.args = list(map(substitute_nodes, node.args))
405 406 407 408 409 410 411

    # replace common subexpressions on all rhs items
    for expr_list in expr_list_list:
        expr_list[-1] = substitute_nodes(expr_list[-1])

def sort_common_subsequences(items):
    """Sort items/subsequences so that all items and subsequences that
Stefan Behnel's avatar
Stefan Behnel committed
412 413 414 415 416 417 418 419 420
    an item contains appear before the item itself.  This is needed
    because each rhs item must only be evaluated once, so its value
    must be evaluated first and then reused when packing sequences
    that contain it.

    This implies a partial order, and the sort must be stable to
    preserve the original order as much as possible, so we use a
    simple insertion sort (which is very fast for short sequences, the
    normal case in practice).
421 422 423 424 425 426 427 428 429 430 431 432
    """
    def contains(seq, x):
        for item in seq:
            if item is x:
                return True
            elif item.is_sequence_constructor and contains(item.args, x):
                return True
        return False
    def lower_than(a,b):
        return b.is_sequence_constructor and contains(b.args, a)

    for pos, item in enumerate(items):
433
        key = item[1] # the ResultRefNode which has already been injected into the sequences
434 435 436 437 438 439 440 441
        new_pos = pos
        for i in xrange(pos-1, -1, -1):
            if lower_than(key, items[i][0]):
                new_pos = i
        if new_pos != pos:
            for i in xrange(pos, new_pos, -1):
                items[i] = items[i-1]
            items[new_pos] = item
442

443 444 445 446 447 448 449 450 451 452 453
def unpack_string_to_character_literals(literal):
    chars = []
    pos = literal.pos
    stype = literal.__class__
    sval = literal.value
    sval_type = sval.__class__
    for char in sval:
        cval = sval_type(char)
        chars.append(stype(pos, value=cval, constant_result=cval))
    return chars

454 455 456 457 458 459 460 461
def flatten_parallel_assignments(input, output):
    #  The input is a list of expression nodes, representing the LHSs
    #  and RHS of one (possibly cascaded) assignment statement.  For
    #  sequence constructors, rearranges the matching parts of both
    #  sides into a list of equivalent assignments between the
    #  individual elements.  This transformation is applied
    #  recursively, so that nested structures get matched as well.
    rhs = input[-1]
462
    if (not (rhs.is_sequence_constructor or isinstance(rhs, ExprNodes.UnicodeNode))
463
        or not sum([lhs.is_sequence_constructor for lhs in input[:-1]])):
464 465 466 467 468
        output.append(input)
        return

    complete_assignments = []

469 470 471 472 473 474
    if rhs.is_sequence_constructor:
        rhs_args = rhs.args
    elif rhs.is_string_literal:
        rhs_args = unpack_string_to_character_literals(rhs)

    rhs_size = len(rhs_args)
Stefan Behnel's avatar
Stefan Behnel committed
475
    lhs_targets = [ [] for _ in xrange(rhs_size) ]
476 477 478 479 480 481 482 483 484
    starred_assignments = []
    for lhs in input[:-1]:
        if not lhs.is_sequence_constructor:
            if lhs.is_starred:
                error(lhs.pos, "starred assignment target must be in a list or tuple")
            complete_assignments.append(lhs)
            continue
        lhs_size = len(lhs.args)
        starred_targets = sum([1 for expr in lhs.args if expr.is_starred])
Stefan Behnel's avatar
Stefan Behnel committed
485 486 487 488 489 490 491 492 493
        if starred_targets > 1:
            error(lhs.pos, "more than 1 starred expression in assignment")
            output.append([lhs,rhs])
            continue
        elif lhs_size - starred_targets > rhs_size:
            error(lhs.pos, "need more than %d value%s to unpack"
                  % (rhs_size, (rhs_size != 1) and 's' or ''))
            output.append([lhs,rhs])
            continue
Stefan Behnel's avatar
Stefan Behnel committed
494
        elif starred_targets:
495
            map_starred_assignment(lhs_targets, starred_assignments,
496
                                   lhs.args, rhs_args)
Stefan Behnel's avatar
Stefan Behnel committed
497 498 499 500 501
        elif lhs_size < rhs_size:
            error(lhs.pos, "too many values to unpack (expected %d, got %d)"
                  % (lhs_size, rhs_size))
            output.append([lhs,rhs])
            continue
502
        else:
Stefan Behnel's avatar
Stefan Behnel committed
503 504
            for targets, expr in zip(lhs_targets, lhs.args):
                targets.append(expr)
505 506 507 508 509 510

    if complete_assignments:
        complete_assignments.append(rhs)
        output.append(complete_assignments)

    # recursively flatten partial assignments
511
    for cascade, rhs in zip(lhs_targets, rhs_args):
512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542
        if cascade:
            cascade.append(rhs)
            flatten_parallel_assignments(cascade, output)

    # recursively flatten starred assignments
    for cascade in starred_assignments:
        if cascade[0].is_sequence_constructor:
            flatten_parallel_assignments(cascade, output)
        else:
            output.append(cascade)

def map_starred_assignment(lhs_targets, starred_assignments, lhs_args, rhs_args):
    # Appends the fixed-position LHS targets to the target list that
    # appear left and right of the starred argument.
    #
    # The starred_assignments list receives a new tuple
    # (lhs_target, rhs_values_list) that maps the remaining arguments
    # (those that match the starred target) to a list.

    # left side of the starred target
    for i, (targets, expr) in enumerate(zip(lhs_targets, lhs_args)):
        if expr.is_starred:
            starred = i
            lhs_remaining = len(lhs_args) - i - 1
            break
        targets.append(expr)
    else:
        raise InternalError("no starred arg found when splitting starred assignment")

    # right side of the starred target
    for i, (targets, expr) in enumerate(zip(lhs_targets[-lhs_remaining:],
Vitja Makarov's avatar
Vitja Makarov committed
543
                                            lhs_args[starred + 1:])):
544 545 546 547 548 549 550 551 552 553 554 555 556 557 558
        targets.append(expr)

    # the starred target itself, must be assigned a (potentially empty) list
    target = lhs_args[starred].target # unpack starred node
    starred_rhs = rhs_args[starred:]
    if lhs_remaining:
        starred_rhs = starred_rhs[:-lhs_remaining]
    if starred_rhs:
        pos = starred_rhs[0].pos
    else:
        pos = target.pos
    starred_assignments.append([
        target, ExprNodes.ListNode(pos=pos, args=starred_rhs)])


559
class PxdPostParse(CythonTransform, SkipDeclarations):
560 561 562
    """
    Basic interpretation/validity checking that should only be
    done on pxd trees.
563 564 565 566 567 568

    A lot of this checking currently happens in the parser; but
    what is listed below happens here.

    - "def" functions are let through only if they fill the
    getbuffer/releasebuffer slots
569

570 571
    - cdef functions are let through only if they are on the
    top level and are declared "inline"
572
    """
573 574
    ERR_INLINE_ONLY = "function definition in pxd file must be declared 'cdef inline'"
    ERR_NOGO_WITH_INLINE = "inline function definition in pxd file cannot be '%s'"
575 576 577 578 579 580 581 582 583 584 585 586 587 588 589

    def __call__(self, node):
        self.scope_type = 'pxd'
        return super(PxdPostParse, self).__call__(node)

    def visit_CClassDefNode(self, node):
        old = self.scope_type
        self.scope_type = 'cclass'
        self.visitchildren(node)
        self.scope_type = old
        return node

    def visit_FuncDefNode(self, node):
        # FuncDefNode always come with an implementation (without
        # an imp they are CVarDefNodes..)
590
        err = self.ERR_INLINE_ONLY
591

592
        if (isinstance(node, Nodes.DefNode) and self.scope_type == 'cclass'
593
            and node.name in ('__getbuffer__', '__releasebuffer__')):
594
            err = None # allow these slots
595

596
        if isinstance(node, Nodes.CFuncDefNode):
597 598
            if (u'inline' in node.modifiers and
                self.scope_type in ('pxd', 'cclass')):
599 600 601 602 603 604 605 606
                node.inline_in_pxd = True
                if node.visibility != 'private':
                    err = self.ERR_NOGO_WITH_INLINE % node.visibility
                elif node.api:
                    err = self.ERR_NOGO_WITH_INLINE % 'api'
                else:
                    err = None # allow inline function
            else:
607 608
                err = self.ERR_INLINE_ONLY

609 610
        if err:
            self.context.nonfatal_error(PostParseError(node.pos, err))
611 612 613
            return None
        else:
            return node
614

615
class InterpretCompilerDirectives(CythonTransform, SkipDeclarations):
616
    """
617
    After parsing, directives can be stored in a number of places:
618 619
    - #cython-comments at the top of the file (stored in ModuleNode)
    - Command-line arguments overriding these
620 621
    - @cython.directivename decorators
    - with cython.directivename: statements
622

623
    This transform is responsible for interpreting these various sources
624
    and store the directive in two ways:
625 626 627 628 629 630 631 632 633 634 635
    - Set the directives attribute of the ModuleNode for global directives.
    - Use a CompilerDirectivesNode to override directives for a subtree.

    (The first one is primarily to not have to modify with the tree
    structure, so that ModuleNode stay on top.)

    The directives are stored in dictionaries from name to value in effect.
    Each such dictionary is always filled in for all possible directives,
    using default values where no value is given by the user.

    The available directives are controlled in Options.py.
636 637 638

    Note that we have to run this prior to analysis, and so some minor
    duplication of functionality has to occur: We manually track cimports
639
    and which names the "cython" module may have been imported to.
640
    """
641
    unop_method_nodes = {
642
        'typeof': ExprNodes.TypeofNode,
643

644 645 646 647 648 649
        'operator.address': ExprNodes.AmpersandNode,
        'operator.dereference': ExprNodes.DereferenceNode,
        'operator.preincrement' : ExprNodes.inc_dec_constructor(True, '++'),
        'operator.predecrement' : ExprNodes.inc_dec_constructor(True, '--'),
        'operator.postincrement': ExprNodes.inc_dec_constructor(False, '++'),
        'operator.postdecrement': ExprNodes.inc_dec_constructor(False, '--'),
650

651
        # For backwards compatability.
652
        'address': ExprNodes.AmpersandNode,
653
    }
Robert Bradshaw's avatar
Robert Bradshaw committed
654 655

    binop_method_nodes = {
656
        'operator.comma'        : ExprNodes.c_binop_constructor(','),
Robert Bradshaw's avatar
Robert Bradshaw committed
657
    }
658

659 660 661
    special_methods = set(['declare', 'union', 'struct', 'typedef',
                           'sizeof', 'cast', 'pointer', 'compiled',
                           'NULL', 'fused_type', 'parallel'])
Stefan Behnel's avatar
Stefan Behnel committed
662
    special_methods.update(unop_method_nodes.keys())
663

Robert Bradshaw's avatar
Robert Bradshaw committed
664
    valid_parallel_directives = set([
Mark Florisson's avatar
Mark Florisson committed
665 666 667
        "parallel",
        "prange",
        "threadid",
668
        #"threadsavailable",
Mark Florisson's avatar
Mark Florisson committed
669 670
    ])

671
    def __init__(self, context, compilation_directive_defaults):
672
        super(InterpretCompilerDirectives, self).__init__(context)
Robert Bradshaw's avatar
Robert Bradshaw committed
673
        self.cython_module_names = set()
Robert Bradshaw's avatar
Robert Bradshaw committed
674
        self.directive_names = {'staticmethod': 'staticmethod'}
Mark Florisson's avatar
Mark Florisson committed
675
        self.parallel_directives = {}
676 677 678 679
        directives = copy.deepcopy(Options.directive_defaults)
        for key, value in compilation_directive_defaults.items():
            directives[unicode(key)] = copy.deepcopy(value)
        self.directives = directives
680

681
    def check_directive_scope(self, pos, directive, scope):
682
        legal_scopes = Options.directive_scopes.get(directive, None)
683 684 685 686 687
        if legal_scopes and scope not in legal_scopes:
            self.context.nonfatal_error(PostParseError(pos, 'The %s compiler directive '
                                        'is not allowed in %s scope' % (directive, scope)))
            return False
        else:
688 689
            if (directive not in Options.directive_defaults
                    and directive not in Options.directive_types):
690
                error(pos, "Invalid directive: '%s'." % (directive,))
691
            return True
692

693
    # Set up processing and handle the cython: comments.
694
    def visit_ModuleNode(self, node):
695
        for key in sorted(node.directive_comments):
696 697
            if not self.check_directive_scope(node.pos, key, 'module'):
                self.wrong_scope_error(node.pos, key, 'module')
698 699
                del node.directive_comments[key]

700 701
        self.module_scope = node.scope

702 703
        self.directives.update(node.directive_comments)
        node.directives = self.directives
Mark Florisson's avatar
Mark Florisson committed
704
        node.parallel_directives = self.parallel_directives
705
        self.visitchildren(node)
706
        node.cython_module_names = self.cython_module_names
707 708
        return node

709 710 711 712 713 714 715
    # The following four functions track imports and cimports that
    # begin with "cython"
    def is_cython_directive(self, name):
        return (name in Options.directive_types or
                name in self.special_methods or
                PyrexTypes.parse_basic_type(name))

Mark Florisson's avatar
Mark Florisson committed
716
    def is_parallel_directive(self, full_name, pos):
Mark Florisson's avatar
Mark Florisson committed
717 718 719 720 721
        """
        Checks to see if fullname (e.g. cython.parallel.prange) is a valid
        parallel directive. If it is a star import it also updates the
        parallel_directives.
        """
Mark Florisson's avatar
Mark Florisson committed
722 723 724
        result = (full_name + ".").startswith("cython.parallel.")

        if result:
Mark Florisson's avatar
Mark Florisson committed
725
            directive = full_name.split('.')
726 727 728
            if full_name == u"cython.parallel":
                self.parallel_directives[u"parallel"] = u"cython.parallel"
            elif full_name == u"cython.parallel.*":
729 730
                for name in self.valid_parallel_directives:
                    self.parallel_directives[name] = u"cython.parallel.%s" % name
Mark Florisson's avatar
Mark Florisson committed
731 732
            elif (len(directive) != 3 or
                  directive[-1] not in self.valid_parallel_directives):
Mark Florisson's avatar
Mark Florisson committed
733 734
                error(pos, "No such directive: %s" % full_name)

735 736
            self.module_scope.use_utility_code(
                UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c"))
737

Mark Florisson's avatar
Mark Florisson committed
738 739
        return result

740 741
    def visit_CImportStatNode(self, node):
        if node.module_name == u"cython":
742
            self.cython_module_names.add(node.as_name or u"cython")
743
        elif node.module_name.startswith(u"cython."):
Mark Florisson's avatar
Mark Florisson committed
744 745 746
            if node.module_name.startswith(u"cython.parallel."):
                error(node.pos, node.module_name + " is not a module")
            if node.module_name == u"cython.parallel":
747
                if node.as_name and node.as_name != u"cython":
Mark Florisson's avatar
Mark Florisson committed
748 749 750 751 752
                    self.parallel_directives[node.as_name] = node.module_name
                else:
                    self.cython_module_names.add(u"cython")
                    self.parallel_directives[
                                    u"cython.parallel"] = node.module_name
753 754
                self.module_scope.use_utility_code(
                    UtilityCode.load_cached("InitThreads", "ModuleSetupCode.c"))
Mark Florisson's avatar
Mark Florisson committed
755
            elif node.as_name:
756
                self.directive_names[node.as_name] = node.module_name[7:]
757
            else:
758
                self.cython_module_names.add(u"cython")
759 760 761
            # if this cimport was a compiler directive, we don't
            # want to leave the cimport node sitting in the tree
            return None
762
        return node
763

764
    def visit_FromCImportStatNode(self, node):
765 766
        if not node.relative_level and (
                node.module_name == u"cython" or node.module_name.startswith(u"cython.")):
767
            submodule = (node.module_name + u".")[7:]
768
            newimp = []
Mark Florisson's avatar
Mark Florisson committed
769

770
            for pos, name, as_name, kind in node.imported_names:
771
                full_name = submodule + name
Mark Florisson's avatar
Mark Florisson committed
772 773 774 775 776 777 778
                qualified_name = u"cython." + full_name

                if self.is_parallel_directive(qualified_name, node.pos):
                    # from cython cimport parallel, or
                    # from cython.parallel cimport parallel, prange, ...
                    self.parallel_directives[as_name or name] = qualified_name
                elif self.is_cython_directive(full_name):
779
                    self.directive_names[as_name or name] = full_name
780 781
                    if kind is not None:
                        self.context.nonfatal_error(PostParseError(pos,
782
                            "Compiler directive imports must be plain imports"))
783 784
                else:
                    newimp.append((pos, name, as_name, kind))
Mark Florisson's avatar
Mark Florisson committed
785

Robert Bradshaw's avatar
Robert Bradshaw committed
786 787
            if not newimp:
                return None
Mark Florisson's avatar
Mark Florisson committed
788

Robert Bradshaw's avatar
Robert Bradshaw committed
789
            node.imported_names = newimp
790
        return node
791

Robert Bradshaw's avatar
Robert Bradshaw committed
792
    def visit_FromImportStatNode(self, node):
793 794
        if (node.module.module_name.value == u"cython") or \
               node.module.module_name.value.startswith(u"cython."):
795
            submodule = (node.module.module_name.value + u".")[7:]
Robert Bradshaw's avatar
Robert Bradshaw committed
796
            newimp = []
797
            for name, name_node in node.items:
798
                full_name = submodule + name
Mark Florisson's avatar
Mark Florisson committed
799 800 801 802
                qualified_name = u"cython." + full_name
                if self.is_parallel_directive(qualified_name, node.pos):
                    self.parallel_directives[name_node.name] = qualified_name
                elif self.is_cython_directive(full_name):
803
                    self.directive_names[name_node.name] = full_name
Robert Bradshaw's avatar
Robert Bradshaw committed
804
                else:
805
                    newimp.append((name, name_node))
Robert Bradshaw's avatar
Robert Bradshaw committed
806 807 808 809 810
            if not newimp:
                return None
            node.items = newimp
        return node

811
    def visit_SingleAssignmentNode(self, node):
812 813 814 815 816 817
        if isinstance(node.rhs, ExprNodes.ImportNode):
            module_name = node.rhs.module_name.value
            is_parallel = (module_name + u".").startswith(u"cython.parallel.")

            if module_name != u"cython" and not is_parallel:
                return node
Mark Florisson's avatar
Mark Florisson committed
818 819 820 821

            module_name = node.rhs.module_name.value
            as_name = node.lhs.name

822
            node = Nodes.CImportStatNode(node.pos,
Mark Florisson's avatar
Mark Florisson committed
823 824
                                         module_name = module_name,
                                         as_name = as_name)
825
            node = self.visit_CImportStatNode(node)
826 827
        else:
            self.visitchildren(node)
828

829
        return node
830

831 832 833
    def visit_NameNode(self, node):
        if node.name in self.cython_module_names:
            node.is_cython_module = True
Robert Bradshaw's avatar
Robert Bradshaw committed
834
        else:
835
            node.cython_attribute = self.directive_names.get(node.name)
836
        return node
837

838
    def try_to_parse_directives(self, node):
839
        # If node is the contents of an directive (in a with statement or
840
        # decorator), returns a list of (directivename, value) pairs.
841
        # Otherwise, returns None
842
        if isinstance(node, ExprNodes.CallNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
843
            self.visit(node.function)
844
            optname = node.function.as_cython_attribute()
845 846 847 848 849 850 851 852 853 854 855 856 857 858 859 860 861 862 863 864 865 866
            if optname:
                directivetype = Options.directive_types.get(optname)
                if directivetype:
                    args, kwds = node.explicit_args_kwds()
                    directives = []
                    key_value_pairs = []
                    if kwds is not None and directivetype is not dict:
                        for keyvalue in kwds.key_value_pairs:
                            key, value = keyvalue
                            sub_optname = "%s.%s" % (optname, key.value)
                            if Options.directive_types.get(sub_optname):
                                directives.append(self.try_to_parse_directive(sub_optname, [value], None, keyvalue.pos))
                            else:
                                key_value_pairs.append(keyvalue)
                        if not key_value_pairs:
                            kwds = None
                        else:
                            kwds.key_value_pairs = key_value_pairs
                        if directives and not kwds and not args:
                            return directives
                    directives.append(self.try_to_parse_directive(optname, args, kwds, node.function.pos))
                    return directives
867
        elif isinstance(node, (ExprNodes.AttributeNode, ExprNodes.NameNode)):
868 869 870 871 872 873 874 875 876 877 878
            self.visit(node)
            optname = node.as_cython_attribute()
            if optname:
                directivetype = Options.directive_types.get(optname)
                if directivetype is bool:
                    return [(optname, True)]
                elif directivetype is None:
                    return [(optname, None)]
                else:
                    raise PostParseError(
                        node.pos, "The '%s' directive should be used as a function call." % optname)
879
        return None
880

881 882
    def try_to_parse_directive(self, optname, args, kwds, pos):
        directivetype = Options.directive_types.get(optname)
883
        if len(args) == 1 and isinstance(args[0], ExprNodes.NoneNode):
884
            return optname, Options.directive_defaults[optname]
885
        elif directivetype is bool:
886
            if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.BoolNode):
887 888 889
                raise PostParseError(pos,
                    'The %s directive takes one compile-time boolean argument' % optname)
            return (optname, args[0].value)
890 891 892 893 894
        elif directivetype is int:
            if kwds is not None or len(args) != 1 or not isinstance(args[0], ExprNodes.IntNode):
                raise PostParseError(pos,
                    'The %s directive takes one compile-time integer argument' % optname)
            return (optname, int(args[0].value))
895
        elif directivetype is str:
896 897
            if kwds is not None or len(args) != 1 or not isinstance(
                    args[0], (ExprNodes.StringNode, ExprNodes.UnicodeNode)):
898 899 900
                raise PostParseError(pos,
                    'The %s directive takes one compile-time string argument' % optname)
            return (optname, str(args[0].value))
901 902 903 904 905
        elif directivetype is type:
            if kwds is not None or len(args) != 1:
                raise PostParseError(pos,
                    'The %s directive takes one type argument' % optname)
            return (optname, args[0])
906 907 908 909 910 911 912 913 914 915
        elif directivetype is dict:
            if len(args) != 0:
                raise PostParseError(pos,
                    'The %s directive takes no prepositional arguments' % optname)
            return optname, dict([(key.value, value) for key, value in kwds.key_value_pairs])
        elif directivetype is list:
            if kwds and len(kwds) != 0:
                raise PostParseError(pos,
                    'The %s directive takes no keyword arguments' % optname)
            return optname, [ str(arg.value) for arg in args ]
916 917 918 919 920
        elif callable(directivetype):
            if kwds is not None or len(args) != 1 or not isinstance(
                    args[0], (ExprNodes.StringNode, ExprNodes.UnicodeNode)):
                raise PostParseError(pos,
                    'The %s directive takes one compile-time string argument' % optname)
Stefan Behnel's avatar
Stefan Behnel committed
921
            return (optname, directivetype(optname, str(args[0].value)))
922 923 924
        else:
            assert False

925 926 927 928 929
    def visit_with_directives(self, body, directives):
        olddirectives = self.directives
        newdirectives = copy.copy(olddirectives)
        newdirectives.update(directives)
        self.directives = newdirectives
930
        assert isinstance(body, Nodes.StatListNode), body
931
        retbody = self.visit_Node(body)
932 933
        directive = Nodes.CompilerDirectivesNode(pos=retbody.pos, body=retbody,
                                                 directives=newdirectives)
934
        self.directives = olddirectives
935
        return directive
936

937
    # Handle decorators
938
    def visit_FuncDefNode(self, node):
939 940 941
        directives = self._extract_directives(node, 'function')
        if not directives:
            return self.visit_Node(node)
942
        body = Nodes.StatListNode(node.pos, stats=[node])
943 944 945
        return self.visit_with_directives(body, directives)

    def visit_CVarDefNode(self, node):
946 947
        directives = self._extract_directives(node, 'function')
        if not directives:
948
            return node
949 950 951
        for name, value in directives.iteritems():
            if name == 'locals':
                node.directive_locals = value
952
            elif name not in ('final', 'staticmethod'):
Stefan Behnel's avatar
Stefan Behnel committed
953 954
                self.context.nonfatal_error(PostParseError(
                    node.pos,
955 956
                    "Cdef functions can only take cython.locals(), "
                    "staticmethod, or final decorators, got %s." % name))
957 958
        body = Nodes.StatListNode(node.pos, stats=[node])
        return self.visit_with_directives(body, directives)
959 960 961 962 963

    def visit_CClassDefNode(self, node):
        directives = self._extract_directives(node, 'cclass')
        if not directives:
            return self.visit_Node(node)
964
        body = Nodes.StatListNode(node.pos, stats=[node])
965 966
        return self.visit_with_directives(body, directives)

967 968 969 970 971 972 973
    def visit_CppClassNode(self, node):
        directives = self._extract_directives(node, 'cppclass')
        if not directives:
            return self.visit_Node(node)
        body = Nodes.StatListNode(node.pos, stats=[node])
        return self.visit_with_directives(body, directives)

974 975 976 977
    def visit_PyClassDefNode(self, node):
        directives = self._extract_directives(node, 'class')
        if not directives:
            return self.visit_Node(node)
978
        body = Nodes.StatListNode(node.pos, stats=[node])
979 980
        return self.visit_with_directives(body, directives)

981 982 983 984 985 986
    def _extract_directives(self, node, scope_name):
        if not node.decorators:
            return {}
        # Split the decorators into two lists -- real decorators and directives
        directives = []
        realdecs = []
Robert Bradshaw's avatar
Robert Bradshaw committed
987
        both = []
988 989 990 991 992
        for dec in node.decorators:
            new_directives = self.try_to_parse_directives(dec.decorator)
            if new_directives is not None:
                for directive in new_directives:
                    if self.check_directive_scope(node.pos, directive[0], scope_name):
Robert Bradshaw's avatar
Robert Bradshaw committed
993 994 995 996 997
                        name, value = directive
                        if self.directives.get(name, object()) != value:
                            directives.append(directive)
                        if directive[0] == 'staticmethod':
                            both.append(dec)
998
            else:
999
                realdecs.append(dec)
1000
        if realdecs and isinstance(node, (Nodes.CFuncDefNode, Nodes.CClassDefNode, Nodes.CVarDefNode)):
1001 1002
            raise PostParseError(realdecs[0].pos, "Cdef functions/classes cannot take arbitrary decorators.")
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
1003
            node.decorators = realdecs + both
1004 1005 1006 1007 1008 1009 1010 1011 1012 1013 1014 1015 1016
        # merge or override repeated directives
        optdict = {}
        directives.reverse() # Decorators coming first take precedence
        for directive in directives:
            name, value = directive
            if name in optdict:
                old_value = optdict[name]
                # keywords and arg lists can be merged, everything
                # else overrides completely
                if isinstance(old_value, dict):
                    old_value.update(value)
                elif isinstance(old_value, list):
                    old_value.extend(value)
1017 1018
                else:
                    optdict[name] = value
1019 1020 1021 1022
            else:
                optdict[name] = value
        return optdict

1023 1024
    # Handle with statements
    def visit_WithStatNode(self, node):
1025 1026 1027 1028 1029 1030 1031 1032
        directive_dict = {}
        for directive in self.try_to_parse_directives(node.manager) or []:
            if directive is not None:
                if node.target is not None:
                    self.context.nonfatal_error(
                        PostParseError(node.pos, "Compiler directive with statements cannot contain 'as'"))
                else:
                    name, value = directive
1033
                    if name in ('nogil', 'gil'):
1034
                        # special case: in pure mode, "with nogil" spells "with cython.nogil"
1035
                        node = Nodes.GILStatNode(node.pos, state = name, body = node.body)
1036
                        return self.visit_Node(node)
1037 1038 1039 1040
                    if self.check_directive_scope(node.pos, name, 'with statement'):
                        directive_dict[name] = value
        if directive_dict:
            return self.visit_with_directives(node.body, directive_dict)
1041
        return self.visit_Node(node)
1042

1043

Mark Florisson's avatar
Mark Florisson committed
1044 1045 1046 1047 1048 1049
class ParallelRangeTransform(CythonTransform, SkipDeclarations):
    """
    Transform cython.parallel stuff. The parallel_directives come from the
    module node, set there by InterpretCompilerDirectives.

        x = cython.parallel.threadavailable()   -> ParallelThreadAvailableNode
1050
        with nogil, cython.parallel.parallel(): -> ParallelWithBlockNode
Mark Florisson's avatar
Mark Florisson committed
1051 1052 1053 1054 1055 1056 1057 1058 1059 1060 1061 1062 1063 1064 1065
            print cython.parallel.threadid()    -> ParallelThreadIdNode
            for i in cython.parallel.prange(...):  -> ParallelRangeNode
                ...
    """

    # a list of names, maps 'cython.parallel.prange' in the code to
    # ['cython', 'parallel', 'prange']
    parallel_directive = None

    # Indicates whether a namenode in an expression is the cython module
    namenode_is_cython_module = False

    # Keep track of whether we are the context manager of a 'with' statement
    in_context_manager_section = False

1066 1067 1068 1069
    # One of 'prange' or 'with parallel'. This is used to disallow closely
    # nested 'with parallel:' blocks
    state = None

Mark Florisson's avatar
Mark Florisson committed
1070 1071 1072 1073 1074 1075 1076 1077 1078 1079 1080 1081 1082 1083 1084 1085 1086 1087 1088 1089 1090 1091 1092 1093 1094 1095
    directive_to_node = {
        u"cython.parallel.parallel": Nodes.ParallelWithBlockNode,
        # u"cython.parallel.threadsavailable": ExprNodes.ParallelThreadsAvailableNode,
        u"cython.parallel.threadid": ExprNodes.ParallelThreadIdNode,
        u"cython.parallel.prange": Nodes.ParallelRangeNode,
    }

    def node_is_parallel_directive(self, node):
        return node.name in self.parallel_directives or node.is_cython_module

    def get_directive_class_node(self, node):
        """
        Figure out which parallel directive was used and return the associated
        Node class.

        E.g. for a cython.parallel.prange() call we return ParallelRangeNode
        """
        if self.namenode_is_cython_module:
            directive = '.'.join(self.parallel_directive)
        else:
            directive = self.parallel_directives[self.parallel_directive[0]]
            directive = '%s.%s' % (directive,
                                   '.'.join(self.parallel_directive[1:]))
            directive = directive.rstrip('.')

        cls = self.directive_to_node.get(directive)
1096 1097
        if cls is None and not (self.namenode_is_cython_module and
                                self.parallel_directive[0] != 'parallel'):
Mark Florisson's avatar
Mark Florisson committed
1098 1099 1100 1101 1102 1103 1104 1105 1106 1107 1108 1109 1110 1111 1112 1113 1114 1115 1116 1117 1118 1119 1120 1121 1122 1123 1124 1125 1126 1127 1128 1129
            error(node.pos, "Invalid directive: %s" % directive)

        self.namenode_is_cython_module = False
        self.parallel_directive = None

        return cls

    def visit_ModuleNode(self, node):
        """
        If any parallel directives were imported, copy them over and visit
        the AST
        """
        if node.parallel_directives:
            self.parallel_directives = node.parallel_directives
            return self.visit_Node(node)

        # No parallel directives were imported, so they can't be used :)
        return node

    def visit_NameNode(self, node):
        if self.node_is_parallel_directive(node):
            self.parallel_directive = [node.name]
            self.namenode_is_cython_module = node.is_cython_module
        return node

    def visit_AttributeNode(self, node):
        self.visitchildren(node)
        if self.parallel_directive:
            self.parallel_directive.append(node.attribute)
        return node

    def visit_CallNode(self, node):
1130
        self.visit(node.function)
Mark Florisson's avatar
Mark Florisson committed
1131 1132 1133 1134 1135 1136 1137 1138 1139 1140 1141 1142 1143 1144 1145
        if not self.parallel_directive:
            return node

        # We are a parallel directive, replace this node with the
        # corresponding ParallelSomethingSomething node

        if isinstance(node, ExprNodes.GeneralCallNode):
            args = node.positional_args.args
            kwargs = node.keyword_args
        else:
            args = node.args
            kwargs = {}

        parallel_directive_class = self.get_directive_class_node(node)
        if parallel_directive_class:
1146 1147
            # Note: in case of a parallel() the body is set by
            # visit_WithStatNode
Mark Florisson's avatar
Mark Florisson committed
1148 1149 1150 1151 1152
            node = parallel_directive_class(node.pos, args=args, kwargs=kwargs)

        return node

    def visit_WithStatNode(self, node):
1153 1154
        "Rewrite with cython.parallel.parallel() blocks"
        newnode = self.visit(node.manager)
Mark Florisson's avatar
Mark Florisson committed
1155

1156
        if isinstance(newnode, Nodes.ParallelWithBlockNode):
1157 1158
            if self.state == 'parallel with':
                error(node.manager.pos,
1159
                      "Nested parallel with blocks are disallowed")
1160 1161

            self.state = 'parallel with'
1162
            body = self.visit(node.body)
1163
            self.state = None
Mark Florisson's avatar
Mark Florisson committed
1164

1165 1166 1167 1168
            newnode.body = body
            return newnode
        elif self.parallel_directive:
            parallel_directive_class = self.get_directive_class_node(node)
1169

1170 1171 1172
            if not parallel_directive_class:
                # There was an error, stop here and now
                return None
Mark Florisson's avatar
Mark Florisson committed
1173

1174 1175 1176
            if parallel_directive_class is Nodes.ParallelWithBlockNode:
                error(node.pos, "The parallel directive must be called")
                return None
Mark Florisson's avatar
Mark Florisson committed
1177

1178 1179
        node.body = self.visit(node.body)
        return node
Mark Florisson's avatar
Mark Florisson committed
1180 1181 1182 1183 1184 1185

    def visit_ForInStatNode(self, node):
        "Rewrite 'for i in cython.parallel.prange(...):'"
        self.visit(node.iterator)
        self.visit(node.target)

1186 1187
        in_prange = isinstance(node.iterator.sequence,
                               Nodes.ParallelRangeNode)
1188
        previous_state = self.state
Mark Florisson's avatar
Mark Florisson committed
1189

1190
        if in_prange:
Mark Florisson's avatar
Mark Florisson committed
1191 1192 1193 1194 1195 1196 1197 1198 1199 1200 1201 1202 1203 1204
            # This will replace the entire ForInStatNode, so copy the
            # attributes
            parallel_range_node = node.iterator.sequence

            parallel_range_node.target = node.target
            parallel_range_node.body = node.body
            parallel_range_node.else_clause = node.else_clause

            node = parallel_range_node

            if not isinstance(node.target, ExprNodes.NameNode):
                error(node.target.pos,
                      "Can only iterate over an iteration variable")

1205
            self.state = 'prange'
Mark Florisson's avatar
Mark Florisson committed
1206

1207 1208 1209
        self.visit(node.body)
        self.state = previous_state
        self.visit(node.else_clause)
Mark Florisson's avatar
Mark Florisson committed
1210 1211 1212 1213 1214
        return node

    def visit(self, node):
        "Visit a node that may be None"
        if node is not None:
1215
            return super(ParallelRangeTransform, self).visit(node)
Mark Florisson's avatar
Mark Florisson committed
1216 1217


1218
class WithTransform(CythonTransform, SkipDeclarations):
1219
    def visit_WithStatNode(self, node):
1220 1221 1222
        self.visitchildren(node, 'body')
        pos = node.pos
        body, target, manager = node.body, node.target, node.manager
1223
        node.enter_call = ExprNodes.SimpleCallNode(
1224 1225 1226 1227 1228
            pos, function=ExprNodes.AttributeNode(
                pos, obj=ExprNodes.CloneNode(manager),
                attribute=EncodedString('__enter__'),
                is_special_lookup=True),
            args=[],
1229 1230
            is_temp=True)

1231 1232
        if target is not None:
            body = Nodes.StatListNode(
Stefan Behnel's avatar
Stefan Behnel committed
1233
                pos, stats=[
1234
                    Nodes.WithTargetAssignmentStatNode(
1235
                        pos, lhs=target, with_node=node),
1236
                    body])
1237

1238 1239
        excinfo_target = ExprNodes.TupleNode(pos, slow=True, args=[
            ExprNodes.ExcValueNode(pos) for _ in range(3)])
1240
        except_clause = Nodes.ExceptClauseNode(
1241 1242
            pos, body=Nodes.IfStatNode(
                pos, if_clauses=[
1243
                    Nodes.IfClauseNode(
1244 1245 1246 1247 1248 1249
                        pos, condition=ExprNodes.NotNode(
                            pos, operand=ExprNodes.WithExitCallNode(
                                pos, with_stat=node,
                                test_if_run=False,
                                args=excinfo_target)),
                        body=Nodes.ReraiseStatNode(pos),
Stefan Behnel's avatar
Stefan Behnel committed
1250 1251
                    ),
                ],
1252 1253 1254 1255
                else_clause=None),
            pattern=None,
            target=None,
            excinfo_target=excinfo_target,
Stefan Behnel's avatar
Stefan Behnel committed
1256
        )
1257 1258

        node.body = Nodes.TryFinallyStatNode(
1259 1260 1261 1262
            pos, body=Nodes.TryExceptStatNode(
                pos, body=body,
                except_clauses=[except_clause],
                else_clause=None,
Stefan Behnel's avatar
Stefan Behnel committed
1263
            ),
1264 1265 1266 1267 1268 1269
            finally_clause=Nodes.ExprStatNode(
                pos, expr=ExprNodes.WithExitCallNode(
                    pos, with_stat=node,
                    test_if_run=True,
                    args=ExprNodes.TupleNode(
                        pos, args=[ExprNodes.NoneNode(pos) for _ in range(3)]
Stefan Behnel's avatar
Stefan Behnel committed
1270
                    ))),
1271
            handle_error_case=False,
Stefan Behnel's avatar
Stefan Behnel committed
1272
        )
1273
        return node
1274

1275 1276 1277
    def visit_ExprNode(self, node):
        # With statements are never inside expressions.
        return node
1278

1279

1280 1281 1282 1283 1284 1285 1286
class DecoratorTransform(ScopeTrackingTransform, SkipDeclarations):
    """Originally, this was the only place where decorators were
    transformed into the corresponding calling code.  Now, this is
    done directly in DefNode and PyClassDefNode to avoid reassignments
    to the function/class name - except for cdef class methods.  For
    those, the reassignment is required as methods are originally
    defined in the PyMethodDef struct.
1287 1288

    The IndirectionNode allows DefNode to override the decorator
1289
    """
1290

1291
    def visit_DefNode(self, func_node):
1292 1293
        scope_type = self.scope_type
        func_node = self.visit_FuncDefNode(func_node)
1294
        if scope_type != 'cclass' or not func_node.decorators:
1295
            return func_node
1296 1297
        return self.handle_decorators(func_node, func_node.decorators,
                                      func_node.name)
1298

1299
    def handle_decorators(self, node, decorators, name):
1300
        decorator_result = ExprNodes.NameNode(node.pos, name = name)
1301
        for decorator in decorators[::-1]:
1302
            decorator_result = ExprNodes.SimpleCallNode(
1303 1304 1305 1306
                decorator.pos,
                function = decorator.decorator,
                args = [decorator_result])

1307 1308
        name_node = ExprNodes.NameNode(node.pos, name = name)
        reassignment = Nodes.SingleAssignmentNode(
1309 1310
            node.pos,
            lhs = name_node,
1311
            rhs = decorator_result)
1312 1313 1314

        reassignment = Nodes.IndirectionNode([reassignment])
        node.decorator_indirection = reassignment
1315
        return [node, reassignment]
1316

1317 1318 1319 1320 1321 1322 1323 1324 1325
class CnameDirectivesTransform(CythonTransform, SkipDeclarations):
    """
    Only part of the CythonUtilityCode pipeline. Must be run before
    DecoratorTransform in case this is a decorator for a cdef class.
    It filters out @cname('my_cname') decorators and rewrites them to
    CnameDecoratorNodes.
    """

    def handle_function(self, node):
1326
        if not getattr(node, 'decorators', None):
1327 1328
            return self.visit_Node(node)

1329 1330 1331 1332 1333 1334 1335 1336 1337 1338 1339 1340 1341 1342 1343 1344 1345 1346 1347 1348 1349
        for i, decorator in enumerate(node.decorators):
            decorator = decorator.decorator

            if (isinstance(decorator, ExprNodes.CallNode) and
                    decorator.function.is_name and
                    decorator.function.name == 'cname'):
                args, kwargs = decorator.explicit_args_kwds()

                if kwargs:
                    raise AssertionError(
                            "cname decorator does not take keyword arguments")

                if len(args) != 1:
                    raise AssertionError(
                            "cname decorator takes exactly one argument")

                if not (args[0].is_literal and
                        args[0].type == Builtin.str_type):
                    raise AssertionError(
                            "argument to cname decorator must be a string literal")

1350
                cname = args[0].compile_time_value(None).decode('UTF-8')
1351 1352 1353 1354 1355
                del node.decorators[i]
                node = Nodes.CnameDecoratorNode(pos=node.pos, node=node,
                                                cname=cname)
                break

1356
        return self.visit_Node(node)
1357

1358 1359
    visit_FuncDefNode = handle_function
    visit_CClassDefNode = handle_function
1360 1361
    visit_CEnumDefNode = handle_function
    visit_CStructOrUnionDefNode = handle_function
1362 1363


1364 1365 1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386 1387 1388 1389 1390 1391 1392 1393 1394 1395 1396 1397 1398 1399 1400
class ForwardDeclareTypes(CythonTransform):

    def visit_CompilerDirectivesNode(self, node):
        env = self.module_scope
        old = env.directives
        env.directives = node.directives
        self.visitchildren(node)
        env.directives = old
        return node

    def visit_ModuleNode(self, node):
        self.module_scope = node.scope
        self.module_scope.directives = node.directives
        self.visitchildren(node)
        return node

    def visit_CDefExternNode(self, node):
        old_cinclude_flag = self.module_scope.in_cinclude
        self.module_scope.in_cinclude = 1
        self.visitchildren(node)
        self.module_scope.in_cinclude = old_cinclude_flag
        return node

    def visit_CEnumDefNode(self, node):
        node.declare(self.module_scope)
        return node

    def visit_CStructOrUnionDefNode(self, node):
        if node.name not in self.module_scope.entries:
            node.declare(self.module_scope)
        return node

    def visit_CClassDefNode(self, node):
        if node.class_name not in self.module_scope.entries:
            node.declare(self.module_scope)
        return node

1401

1402
class AnalyseDeclarationsTransform(EnvTransform):
1403

1404 1405 1406 1407 1408 1409
    basic_property = TreeFragment(u"""
property NAME:
    def __get__(self):
        return ATTR
    def __set__(self, value):
        ATTR = value
1410
    """, level='c_class', pipeline=[NormalizeTree(None)])
1411 1412 1413 1414 1415 1416 1417 1418
    basic_pyobject_property = TreeFragment(u"""
property NAME:
    def __get__(self):
        return ATTR
    def __set__(self, value):
        ATTR = value
    def __del__(self):
        ATTR = None
1419
    """, level='c_class', pipeline=[NormalizeTree(None)])
1420 1421 1422 1423
    basic_property_ro = TreeFragment(u"""
property NAME:
    def __get__(self):
        return ATTR
1424
    """, level='c_class', pipeline=[NormalizeTree(None)])
1425

1426 1427 1428 1429 1430 1431 1432 1433 1434 1435 1436 1437 1438
    struct_or_union_wrapper = TreeFragment(u"""
cdef class NAME:
    cdef TYPE value
    def __init__(self, MEMBER=None):
        cdef int count
        count = 0
        INIT_ASSIGNMENTS
        if IS_UNION and count > 1:
            raise ValueError, "At most one union member should be specified."
    def __str__(self):
        return STR_FORMAT % MEMBER_TUPLE
    def __repr__(self):
        return REPR_FORMAT % MEMBER_TUPLE
1439
    """, pipeline=[NormalizeTree(None)])
1440 1441 1442 1443 1444

    init_assignment = TreeFragment(u"""
if VALUE is not None:
    ATTR = VALUE
    count += 1
1445
    """, pipeline=[NormalizeTree(None)])
1446

1447
    fused_function = None
1448
    in_lambda = 0
1449

1450
    def __call__(self, root):
1451
        # needed to determine if a cdef var is declared after it's used.
1452
        self.seen_vars_stack = []
1453
        self.fused_error_funcs = set()
1454 1455 1456
        super_class = super(AnalyseDeclarationsTransform, self)
        self._super_visit_FuncDefNode = super_class.visit_FuncDefNode
        return super_class.__call__(root)
1457

1458
    def visit_NameNode(self, node):
1459
        self.seen_vars_stack[-1].add(node.name)
1460 1461
        return node

1462
    def visit_ModuleNode(self, node):
Robert Bradshaw's avatar
Robert Bradshaw committed
1463
        self.seen_vars_stack.append(set())
1464
        node.analyse_declarations(self.current_env())
1465
        self.visitchildren(node)
1466
        self.seen_vars_stack.pop()
1467
        return node
Stefan Behnel's avatar
Stefan Behnel committed
1468 1469

    def visit_LambdaNode(self, node):
1470
        self.in_lambda += 1
1471
        node.analyse_declarations(self.current_env())
Stefan Behnel's avatar
Stefan Behnel committed
1472
        self.visitchildren(node)
1473
        self.in_lambda -= 1
Stefan Behnel's avatar
Stefan Behnel committed
1474 1475
        return node

1476 1477
    def visit_CClassDefNode(self, node):
        node = self.visit_ClassDefNode(node)
1478
        if node.scope and node.scope.implemented and node.body:
1479 1480 1481 1482 1483 1484 1485 1486 1487 1488
            stats = []
            for entry in node.scope.var_entries:
                if entry.needs_property:
                    property = self.create_Property(entry)
                    property.analyse_declarations(node.scope)
                    self.visit(property)
                    stats.append(property)
            if stats:
                node.body.stats += stats
        return node
1489

1490 1491 1492 1493 1494 1495 1496 1497 1498 1499 1500 1501 1502 1503 1504 1505 1506 1507 1508 1509 1510 1511 1512 1513 1514 1515 1516 1517 1518 1519 1520 1521 1522 1523 1524 1525 1526 1527 1528 1529 1530 1531 1532 1533 1534 1535 1536
    def _handle_fused_def_decorators(self, old_decorators, env, node):
        """
        Create function calls to the decorators and reassignments to
        the function.
        """
        # Delete staticmethod and classmethod decorators, this is
        # handled directly by the fused function object.
        decorators = []
        for decorator in old_decorators:
            func = decorator.decorator
            if (not func.is_name or
                func.name not in ('staticmethod', 'classmethod') or
                env.lookup_here(func.name)):
                # not a static or classmethod
                decorators.append(decorator)

        if decorators:
            transform = DecoratorTransform(self.context)
            def_node = node.node
            _, reassignments = transform.handle_decorators(
                def_node, decorators, def_node.name)
            reassignments.analyse_declarations(env)
            node = [node, reassignments]

        return node

    def _handle_def(self, decorators, env, node):
        "Handle def or cpdef fused functions"
        # Create PyCFunction nodes for each specialization
        node.stats.insert(0, node.py_func)
        node.py_func = self.visit(node.py_func)
        node.update_fused_defnode_entry(env)
        pycfunc = ExprNodes.PyCFunctionNode.from_defnode(node.py_func,
                                                         True)
        pycfunc = ExprNodes.ProxyNode(pycfunc.coerce_to_temp(env))
        node.resulting_fused_function = pycfunc
        # Create assignment node for our def function
        node.fused_func_assignment = self._create_assignment(
            node.py_func, ExprNodes.CloneNode(pycfunc), env)

        if decorators:
            node = self._handle_fused_def_decorators(decorators, env, node)

        return node

    def _create_fused_function(self, env, node):
        "Create a fused function for a DefNode with fused arguments"
1537
        from . import FusedNode
1538 1539 1540 1541 1542 1543 1544 1545 1546 1547 1548 1549 1550 1551 1552 1553 1554 1555 1556 1557 1558 1559 1560 1561 1562 1563 1564 1565 1566 1567 1568 1569 1570 1571 1572 1573 1574 1575 1576 1577 1578 1579 1580 1581 1582 1583 1584 1585 1586

        if self.fused_function or self.in_lambda:
            if self.fused_function not in self.fused_error_funcs:
                if self.in_lambda:
                    error(node.pos, "Fused lambdas not allowed")
                else:
                    error(node.pos, "Cannot nest fused functions")

            self.fused_error_funcs.add(self.fused_function)

            node.body = Nodes.PassStatNode(node.pos)
            for arg in node.args:
                if arg.type.is_fused:
                    arg.type = arg.type.get_fused_types()[0]

            return node

        decorators = getattr(node, 'decorators', None)
        node = FusedNode.FusedCFuncDefNode(node, env)
        self.fused_function = node
        self.visitchildren(node)
        self.fused_function = None
        if node.py_func:
            node = self._handle_def(decorators, env, node)

        return node

    def _handle_nogil_cleanup(self, lenv, node):
        "Handle cleanup for 'with gil' blocks in nogil functions."
        if lenv.nogil and lenv.has_with_gil_block:
            # Acquire the GIL for cleanup in 'nogil' functions, by wrapping
            # the entire function body in try/finally.
            # The corresponding release will be taken care of by
            # Nodes.FuncDefNode.generate_function_definitions()
            node.body = Nodes.NogilTryFinallyStatNode(
                node.body.pos,
                body=node.body,
                finally_clause=Nodes.EnsureGILNode(node.body.pos))

    def _handle_fused(self, node):
        if node.is_generator and node.has_fused_arguments:
            node.has_fused_arguments = False
            error(node.pos, "Fused generators not supported")
            node.gbody = Nodes.StatListNode(node.pos,
                                            stats=[],
                                            body=Nodes.PassStatNode(node.pos))

        return node.has_fused_arguments

1587
    def visit_FuncDefNode(self, node):
1588
        """
Stefan Behnel's avatar
Stefan Behnel committed
1589 1590 1591 1592 1593 1594 1595 1596
        Analyse a function and its body, as that hasn't happend yet.  Also
        analyse the directive_locals set by @cython.locals().

        Then, if we are a function with fused arguments, replace the function
        (after it has declared itself in the symbol table!) with a
        FusedCFuncDefNode, and analyse its children (which are in turn normal
        functions). If we're a normal function, just analyse the body of the
        function.
1597
        """
1598
        env = self.current_env()
1599

Robert Bradshaw's avatar
Robert Bradshaw committed
1600
        self.seen_vars_stack.append(set())
1601
        lenv = node.local_scope
1602
        node.declare_arguments(lenv)
1603

Stefan Behnel's avatar
Stefan Behnel committed
1604
        # @cython.locals(...)
1605 1606 1607 1608 1609 1610 1611
        for var, type_node in node.directive_locals.items():
            if not lenv.lookup_here(var):   # don't redeclare args
                type = type_node.analyse_as_type(lenv)
                if type:
                    lenv.declare_var(var, type, type_node.pos)
                else:
                    error(type_node.pos, "Not a type")
1612

1613 1614
        if self._handle_fused(node):
            node = self._create_fused_function(env, node)
1615 1616
        else:
            node.body.analyse_declarations(lenv)
1617
            self._handle_nogil_cleanup(lenv, node)
1618
            self._super_visit_FuncDefNode(node)
1619

1620
        self.seen_vars_stack.pop()
1621
        return node
1622

1623 1624
    def visit_DefNode(self, node):
        node = self.visit_FuncDefNode(node)
1625
        env = self.current_env()
1626
        if (not isinstance(node, Nodes.DefNode) or
Stefan Behnel's avatar
Stefan Behnel committed
1627 1628
                node.fused_py_func or node.is_generator_body or
                not node.needs_assignment_synthesis(env)):
1629 1630 1631
            return node
        return [node, self._synthesize_assignment(node, env)]

1632 1633 1634
    def visit_GeneratorBodyDefNode(self, node):
        return self.visit_FuncDefNode(node)

1635 1636 1637 1638 1639 1640 1641 1642 1643 1644 1645 1646
    def _synthesize_assignment(self, node, env):
        # Synthesize assignment node and put it right after defnode
        genv = env
        while genv.is_py_class_scope or genv.is_c_class_scope:
            genv = genv.outer_scope

        if genv.is_closure_scope:
            rhs = node.py_cfunc_node = ExprNodes.InnerFunctionNode(
                node.pos, def_node=node,
                pymethdef_cname=node.entry.pymethdef_cname,
                code_object=ExprNodes.CodeObjectNode(node))
        else:
1647 1648
            binding = self.current_directives.get('binding')
            rhs = ExprNodes.PyCFunctionNode.from_defnode(node, binding)
1649
            node.code_object = rhs.code_object
1650 1651 1652 1653 1654

        if env.is_py_class_scope:
            rhs.binding = True

        node.is_cyfunction = rhs.binding
1655
        return self._create_assignment(node, rhs, env)
1656

1657 1658 1659
    def _create_assignment(self, def_node, rhs, env):
        if def_node.decorators:
            for decorator in def_node.decorators[::-1]:
1660 1661 1662 1663
                rhs = ExprNodes.SimpleCallNode(
                    decorator.pos,
                    function = decorator.decorator,
                    args = [rhs])
1664
            def_node.decorators = None
1665 1666

        assmt = Nodes.SingleAssignmentNode(
1667 1668
            def_node.pos,
            lhs=ExprNodes.NameNode(def_node.pos, name=def_node.name),
1669 1670 1671 1672
            rhs=rhs)
        assmt.analyse_declarations(env)
        return assmt

1673
    def visit_ScopedExprNode(self, node):
1674
        env = self.current_env()
1675
        node.analyse_declarations(env)
Stefan Behnel's avatar
Stefan Behnel committed
1676
        # the node may or may not have a local scope
1677
        if node.has_local_scope:
Robert Bradshaw's avatar
Robert Bradshaw committed
1678
            self.seen_vars_stack.append(set(self.seen_vars_stack[-1]))
1679
            self.enter_scope(node, node.expr_scope)
1680
            node.analyse_scoped_declarations(node.expr_scope)
Stefan Behnel's avatar
Stefan Behnel committed
1681
            self.visitchildren(node)
1682
            self.exit_scope()
Stefan Behnel's avatar
Stefan Behnel committed
1683
            self.seen_vars_stack.pop()
1684
        else:
1685
            node.analyse_scoped_declarations(env)
Stefan Behnel's avatar
Stefan Behnel committed
1686
            self.visitchildren(node)
1687 1688
        return node

1689 1690
    def visit_TempResultFromStatNode(self, node):
        self.visitchildren(node)
1691
        node.analyse_declarations(self.current_env())
1692 1693
        return node

1694 1695 1696 1697 1698
    def visit_CppClassNode(self, node):
        if node.visibility == 'extern':
            return None
        else:
            return self.visit_ClassDefNode(node)
1699

1700
    def visit_CStructOrUnionDefNode(self, node):
1701
        # Create a wrapper node if needed.
1702 1703 1704
        # We want to use the struct type information (so it can't happen
        # before this phase) but also create new objects to be declared
        # (so it can't happen later).
1705
        # Note that we don't return the original node, as it is
1706 1707 1708
        # never used after this phase.
        if True: # private (default)
            return None
1709

1710 1711 1712 1713 1714 1715 1716 1717 1718 1719
        self_value = ExprNodes.AttributeNode(
            pos = node.pos,
            obj = ExprNodes.NameNode(pos=node.pos, name=u"self"),
            attribute = EncodedString(u"value"))
        var_entries = node.entry.type.scope.var_entries
        attributes = []
        for entry in var_entries:
            attributes.append(ExprNodes.AttributeNode(pos = entry.pos,
                                                      obj = self_value,
                                                      attribute = entry.name))
1720 1721 1722 1723 1724 1725 1726 1727 1728 1729 1730 1731 1732 1733 1734 1735 1736 1737 1738 1739 1740 1741 1742 1743 1744 1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756
        # __init__ assignments
        init_assignments = []
        for entry, attr in zip(var_entries, attributes):
            # TODO: branch on visibility
            init_assignments.append(self.init_assignment.substitute({
                    u"VALUE": ExprNodes.NameNode(entry.pos, name = entry.name),
                    u"ATTR": attr,
                }, pos = entry.pos))

        # create the class
        str_format = u"%s(%s)" % (node.entry.type.name, ("%s, " * len(attributes))[:-2])
        wrapper_class = self.struct_or_union_wrapper.substitute({
            u"INIT_ASSIGNMENTS": Nodes.StatListNode(node.pos, stats = init_assignments),
            u"IS_UNION": ExprNodes.BoolNode(node.pos, value = not node.entry.type.is_struct),
            u"MEMBER_TUPLE": ExprNodes.TupleNode(node.pos, args=attributes),
            u"STR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format)),
            u"REPR_FORMAT": ExprNodes.StringNode(node.pos, value = EncodedString(str_format.replace("%s", "%r"))),
        }, pos = node.pos).stats[0]
        wrapper_class.class_name = node.name
        wrapper_class.shadow = True
        class_body = wrapper_class.body.stats

        # fix value type
        assert isinstance(class_body[0].base_type, Nodes.CSimpleBaseTypeNode)
        class_body[0].base_type.name = node.name

        # fix __init__ arguments
        init_method = class_body[1]
        assert isinstance(init_method, Nodes.DefNode) and init_method.name == '__init__'
        arg_template = init_method.args[1]
        if not node.entry.type.is_struct:
            arg_template.kw_only = True
        del init_method.args[1]
        for entry, attr in zip(var_entries, attributes):
            arg = copy.deepcopy(arg_template)
            arg.declarator.name = entry.name
            init_method.args.append(arg)
Robert Bradshaw's avatar
Robert Bradshaw committed
1757

1758
        # setters/getters
1759 1760 1761 1762 1763 1764 1765 1766 1767 1768
        for entry, attr in zip(var_entries, attributes):
            # TODO: branch on visibility
            if entry.type.is_pyobject:
                template = self.basic_pyobject_property
            else:
                template = self.basic_property
            property = template.substitute({
                    u"ATTR": attr,
                }, pos = entry.pos).stats[0]
            property.name = entry.name
1769
            wrapper_class.body.stats.append(property)
Robert Bradshaw's avatar
Robert Bradshaw committed
1770

1771
        wrapper_class.analyse_declarations(self.current_env())
1772
        return self.visit_CClassDefNode(wrapper_class)
1773

1774 1775 1776 1777
    # Some nodes are no longer needed after declaration
    # analysis and can be dropped. The analysis was performed
    # on these nodes in a seperate recursive process from the
    # enclosing function or module, so we can simply drop them.
1778
    def visit_CDeclaratorNode(self, node):
1779 1780
        # necessary to ensure that all CNameDeclaratorNodes are visited.
        self.visitchildren(node)
1781
        return node
1782

1783 1784 1785 1786 1787
    def visit_CTypeDefNode(self, node):
        return node

    def visit_CBaseTypeNode(self, node):
        return None
1788

1789
    def visit_CEnumDefNode(self, node):
1790 1791 1792 1793
        if node.visibility == 'public':
            return node
        else:
            return None
1794

1795
    def visit_CNameDeclaratorNode(self, node):
1796
        if node.name in self.seen_vars_stack[-1]:
1797
            entry = self.current_env().lookup(node.name)
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1798 1799
            if (entry is None or entry.visibility != 'extern'
                and not entry.scope.is_c_class_scope):
1800
                warning(node.pos, "cdef variable '%s' declared after it is used" % node.name, 2)
1801 1802 1803
        self.visitchildren(node)
        return node

1804
    def visit_CVarDefNode(self, node):
1805 1806
        # to ensure all CNameDeclaratorNodes are visited.
        self.visitchildren(node)
1807
        return None
1808

1809
    def visit_CnameDecoratorNode(self, node):
1810 1811
        child_node = self.visit(node.node)
        if not child_node:
1812
            return None
1813 1814 1815 1816
        if type(child_node) is list: # Assignment synthesized
            node.child_node = child_node[0]
            return [node] + child_node[1:]
        node.node = child_node
1817 1818
        return node

1819
    def create_Property(self, entry):
1820
        if entry.visibility == 'public':
1821 1822 1823 1824
            if entry.type.is_pyobject:
                template = self.basic_pyobject_property
            else:
                template = self.basic_property
1825 1826
        elif entry.visibility == 'readonly':
            template = self.basic_property_ro
1827
        property = template.substitute({
1828
                u"ATTR": ExprNodes.AttributeNode(pos=entry.pos,
1829
                                                 obj=ExprNodes.NameNode(pos=entry.pos, name="self"),
1830
                                                 attribute=entry.name),
1831 1832
            }, pos=entry.pos).stats[0]
        property.name = entry.name
1833
        property.doc = entry.doc
1834
        return property
1835

1836

1837 1838 1839 1840 1841 1842 1843 1844
class CalculateQualifiedNamesTransform(EnvTransform):
    """
    Calculate and store the '__qualname__' and the global
    module name on some nodes.
    """
    def visit_ModuleNode(self, node):
        self.module_name = self.global_scope().qualified_name
        self.qualified_name = []
1845 1846 1847
        _super = super(CalculateQualifiedNamesTransform, self)
        self._super_visit_FuncDefNode = _super.visit_FuncDefNode
        self._super_visit_ClassDefNode = _super.visit_ClassDefNode
1848 1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
        self.visitchildren(node)
        return node

    def _set_qualname(self, node, name=None):
        if name:
            qualname = self.qualified_name[:]
            qualname.append(name)
        else:
            qualname = self.qualified_name
        node.qualname = EncodedString('.'.join(qualname))
        node.module_name = self.module_name

1860 1861 1862
    def _append_entry(self, entry):
        if entry.is_pyglobal and not entry.is_pyclass_attr:
            self.qualified_name = [entry.name]
1863
        else:
1864
            self.qualified_name.append(entry.name)
1865 1866

    def visit_ClassNode(self, node):
1867 1868 1869
        self._set_qualname(node, node.name)
        self.visitchildren(node)
        return node
1870 1871

    def visit_PyClassNamespaceNode(self, node):
1872
        # class name was already added by parent node
1873 1874 1875
        self._set_qualname(node)
        self.visitchildren(node)
        return node
1876 1877

    def visit_PyCFunctionNode(self, node):
1878 1879 1880
        self._set_qualname(node, node.def_node.name)
        self.visitchildren(node)
        return node
1881

1882
    def visit_DefNode(self, node):
1883
        self._set_qualname(node, node.name)
1884 1885 1886
        return self.visit_FuncDefNode(node)

    def visit_FuncDefNode(self, node):
1887
        orig_qualified_name = self.qualified_name[:]
1888 1889 1890 1891
        if getattr(node, 'name', None) == '<lambda>':
            self.qualified_name.append('<lambda>')
        else:
            self._append_entry(node.entry)
1892
        self.qualified_name.append('<locals>')
1893
        self._super_visit_FuncDefNode(node)
1894 1895 1896 1897 1898
        self.qualified_name = orig_qualified_name
        return node

    def visit_ClassDefNode(self, node):
        orig_qualified_name = self.qualified_name[:]
1899 1900 1901 1902
        entry = (getattr(node, 'entry', None) or             # PyClass
                 self.current_env().lookup_here(node.name))  # CClass
        self._append_entry(entry)
        self._super_visit_ClassDefNode(node)
1903 1904 1905 1906
        self.qualified_name = orig_qualified_name
        return node


1907
class AnalyseExpressionsTransform(CythonTransform):
1908

1909
    def visit_ModuleNode(self, node):
1910
        node.scope.infer_types()
1911
        node.body = node.body.analyse_expressions(node.scope)
1912 1913
        self.visitchildren(node)
        return node
1914

1915
    def visit_FuncDefNode(self, node):
1916
        node.local_scope.infer_types()
1917
        node.body = node.body.analyse_expressions(node.local_scope)
1918 1919
        self.visitchildren(node)
        return node
1920 1921

    def visit_ScopedExprNode(self, node):
1922
        if node.has_local_scope:
1923
            node.expr_scope.infer_types()
1924
            node = node.analyse_scoped_expressions(node.expr_scope)
1925 1926
        self.visitchildren(node)
        return node
1927

1928 1929 1930
    def visit_IndexNode(self, node):
        """
        Replace index nodes used to specialize cdef functions with fused
Mark Florisson's avatar
Mark Florisson committed
1931 1932 1933
        argument types with the Attribute- or NameNode referring to the
        function. We then need to copy over the specialization properties to
        the attribute or name node.
1934 1935 1936 1937

        Because the indexing might be a Python indexing operation on a fused
        function, or (usually) a Cython indexing operation, we need to
        re-analyse the types.
1938 1939 1940
        """
        self.visit_Node(node)

1941
        if node.is_fused_index and not node.type.is_error:
1942
            node = node.base
1943
        elif node.memslice_ellipsis_noop:
1944 1945
            # memoryviewslice[...] expression, drop the IndexNode
            node = node.base
1946

1947
        return node
1948

1949

1950 1951 1952 1953 1954 1955 1956 1957 1958 1959 1960 1961 1962 1963 1964 1965 1966 1967 1968 1969 1970 1971
class FindInvalidUseOfFusedTypes(CythonTransform):

    def visit_FuncDefNode(self, node):
        # Errors related to use in functions with fused args will already
        # have been detected
        if not node.has_fused_arguments:
            if not node.is_generator_body and node.return_type.is_fused:
                error(node.pos, "Return type is not specified as argument type")
            else:
                self.visitchildren(node)

        return node

    def visit_ExprNode(self, node):
        if node.type and node.type.is_fused:
            error(node.pos, "Invalid use of fused types, type cannot be specialized")
        else:
            self.visitchildren(node)

        return node


1972
class ExpandInplaceOperators(EnvTransform):
1973

1974 1975 1976 1977 1978 1979
    def visit_InPlaceAssignmentNode(self, node):
        lhs = node.lhs
        rhs = node.rhs
        if lhs.type.is_cpp_class:
            # No getting around this exact operator here.
            return node
1980
        if isinstance(lhs, ExprNodes.IndexNode) and lhs.is_buffer_access:
1981 1982 1983
            # There is code to handle this case.
            return node

Robert Bradshaw's avatar
Robert Bradshaw committed
1984
        env = self.current_env()
1985
        def side_effect_free_reference(node, setting=False):
1986
            if isinstance(node, ExprNodes.NameNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
1987 1988
                return node, []
            elif node.type.is_pyobject and not setting:
1989 1990
                node = LetRefNode(node)
                return node, [node]
1991
            elif isinstance(node, ExprNodes.IndexNode):
1992
                if node.is_buffer_access:
1993
                    raise ValueError("Buffer access")
1994 1995
                base, temps = side_effect_free_reference(node.base)
                index = LetRefNode(node.index)
1996 1997
                return ExprNodes.IndexNode(node.pos, base=base, index=index), temps + [index]
            elif isinstance(node, ExprNodes.AttributeNode):
1998
                obj, temps = side_effect_free_reference(node.obj)
1999
                return ExprNodes.AttributeNode(node.pos, obj=obj, attribute=node.attribute), temps
2000 2001 2002 2003 2004 2005 2006 2007
            else:
                node = LetRefNode(node)
                return node, [node]
        try:
            lhs, let_ref_nodes = side_effect_free_reference(lhs, setting=True)
        except ValueError:
            return node
        dup = lhs.__class__(**lhs.__dict__)
2008
        binop = ExprNodes.binop_node(node.pos,
2009 2010 2011 2012
                                     operator = node.operator,
                                     operand1 = dup,
                                     operand2 = rhs,
                                     inplace=True)
Robert Bradshaw's avatar
Robert Bradshaw committed
2013 2014 2015
        # Manually analyse types for new node.
        lhs.analyse_target_types(env)
        dup.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
2016
        binop.analyse_operation(env)
2017
        node = Nodes.SingleAssignmentNode(
2018
            node.pos,
2019 2020
            lhs = lhs,
            rhs=binop.coerce_to(lhs.type, env))
2021 2022 2023 2024 2025 2026 2027 2028 2029 2030
        # Use LetRefNode to avoid side effects.
        let_ref_nodes.reverse()
        for t in let_ref_nodes:
            node = LetNode(t, node)
        return node

    def visit_ExprNode(self, node):
        # In-place assignments can't happen within an expression.
        return node

2031

Haoyu Bai's avatar
Haoyu Bai committed
2032 2033 2034 2035 2036 2037 2038
class AdjustDefByDirectives(CythonTransform, SkipDeclarations):
    """
    Adjust function and class definitions by the decorator directives:

    @cython.cfunc
    @cython.cclass
    @cython.ccall
2039
    @cython.inline
Haoyu Bai's avatar
Haoyu Bai committed
2040 2041 2042 2043 2044 2045 2046 2047 2048 2049 2050 2051 2052 2053 2054 2055
    """

    def visit_ModuleNode(self, node):
        self.directives = node.directives
        self.in_py_class = False
        self.visitchildren(node)
        return node

    def visit_CompilerDirectivesNode(self, node):
        old_directives = self.directives
        self.directives = node.directives
        self.visitchildren(node)
        self.directives = old_directives
        return node

    def visit_DefNode(self, node):
2056 2057 2058
        modifiers = []
        if 'inline' in self.directives:
            modifiers.append('inline')
Haoyu Bai's avatar
Haoyu Bai committed
2059
        if 'ccall' in self.directives:
2060 2061
            node = node.as_cfunction(
                overridable=True, returns=self.directives.get('returns'), modifiers=modifiers)
Haoyu Bai's avatar
Haoyu Bai committed
2062
            return self.visit(node)
Haoyu Bai's avatar
Haoyu Bai committed
2063 2064 2065 2066
        if 'cfunc' in self.directives:
            if self.in_py_class:
                error(node.pos, "cfunc directive is not allowed here")
            else:
2067 2068
                node = node.as_cfunction(
                    overridable=False, returns=self.directives.get('returns'), modifiers=modifiers)
Haoyu Bai's avatar
Haoyu Bai committed
2069
                return self.visit(node)
2070 2071
        if 'inline' in modifiers:
            error(node.pos, "Python functions cannot be declared 'inline'")
Haoyu Bai's avatar
Haoyu Bai committed
2072 2073 2074 2075
        self.visitchildren(node)
        return node

    def visit_PyClassDefNode(self, node):
2076 2077 2078 2079 2080 2081 2082 2083 2084
        if 'cclass' in self.directives:
            node = node.as_cclass()
            return self.visit(node)
        else:
            old_in_pyclass = self.in_py_class
            self.in_py_class = True
            self.visitchildren(node)
            self.in_py_class = old_in_pyclass
            return node
Haoyu Bai's avatar
Haoyu Bai committed
2085 2086 2087 2088 2089 2090 2091

    def visit_CClassDefNode(self, node):
        old_in_pyclass = self.in_py_class
        self.in_py_class = False
        self.visitchildren(node)
        self.in_py_class = old_in_pyclass
        return node
2092

2093

2094 2095
class AlignFunctionDefinitions(CythonTransform):
    """
2096 2097
    This class takes the signatures from a .pxd file and applies them to
    the def methods in a .py file.
2098
    """
2099

2100 2101
    def visit_ModuleNode(self, node):
        self.scope = node.scope
2102
        self.directives = node.directives
2103
        self.imported_names = set()  # hack, see visit_FromImportStatNode()
2104 2105
        self.visitchildren(node)
        return node
2106

2107 2108 2109 2110 2111
    def visit_PyClassDefNode(self, node):
        pxd_def = self.scope.lookup(node.name)
        if pxd_def:
            if pxd_def.is_cclass:
                return self.visit_CClassDefNode(node.as_cclass(), pxd_def)
2112
            elif not pxd_def.scope or not pxd_def.scope.is_builtin_scope:
2113
                error(node.pos, "'%s' redeclared" % node.name)
2114 2115
                if pxd_def.pos:
                    error(pxd_def.pos, "previous declaration here")
2116
                return None
2117
        return node
2118

2119 2120 2121 2122 2123 2124 2125 2126 2127 2128
    def visit_CClassDefNode(self, node, pxd_def=None):
        if pxd_def is None:
            pxd_def = self.scope.lookup(node.class_name)
        if pxd_def:
            outer_scope = self.scope
            self.scope = pxd_def.type.scope
        self.visitchildren(node)
        if pxd_def:
            self.scope = outer_scope
        return node
2129

2130 2131
    def visit_DefNode(self, node):
        pxd_def = self.scope.lookup(node.name)
2132
        if pxd_def and (not pxd_def.scope or not pxd_def.scope.is_builtin_scope):
2133
            if not pxd_def.is_cfunction:
2134
                error(node.pos, "'%s' redeclared" % node.name)
2135 2136
                if pxd_def.pos:
                    error(pxd_def.pos, "previous declaration here")
2137
                return None
2138
            node = node.as_cfunction(pxd_def)
2139
        elif (self.scope.is_module_scope and self.directives['auto_cpdef']
2140
              and not node.name in self.imported_names
2141
              and node.is_cdef_func_compatible()):
2142
            # FIXME: cpdef-ing should be done in analyse_declarations()
2143
            node = node.as_cfunction(scope=self.scope)
2144
        # Enable this when nested cdef functions are allowed.
2145 2146
        # self.visitchildren(node)
        return node
2147

2148 2149 2150 2151 2152 2153 2154 2155 2156
    def visit_FromImportStatNode(self, node):
        # hack to prevent conditional import fallback functions from
        # being cdpef-ed (global Python variables currently conflict
        # with imports)
        if self.scope.is_module_scope:
            for name, _ in node.items:
                self.imported_names.add(name)
        return node

2157 2158 2159 2160
    def visit_ExprNode(self, node):
        # ignore lambdas and everything else that appears in expressions
        return node

2161

2162 2163 2164 2165 2166 2167 2168 2169 2170 2171 2172 2173 2174 2175 2176 2177 2178 2179 2180 2181 2182 2183 2184 2185 2186 2187 2188 2189 2190 2191 2192 2193
class RemoveUnreachableCode(CythonTransform):
    def visit_StatListNode(self, node):
        if not self.current_directives['remove_unreachable']:
            return node
        self.visitchildren(node)
        for idx, stat in enumerate(node.stats):
            idx += 1
            if stat.is_terminator:
                if idx < len(node.stats):
                    if self.current_directives['warn.unreachable']:
                        warning(node.stats[idx].pos, "Unreachable code", 2)
                    node.stats = node.stats[:idx]
                node.is_terminator = True
                break
        return node

    def visit_IfClauseNode(self, node):
        self.visitchildren(node)
        if node.body.is_terminator:
            node.is_terminator = True
        return node

    def visit_IfStatNode(self, node):
        self.visitchildren(node)
        if node.else_clause and node.else_clause.is_terminator:
            for clause in node.if_clauses:
                if not clause.is_terminator:
                    break
            else:
                node.is_terminator = True
        return node

2194 2195
    def visit_TryExceptStatNode(self, node):
        self.visitchildren(node)
2196 2197 2198
        if node.body.is_terminator and node.else_clause:
            if self.current_directives['warn.unreachable']:
                warning(node.else_clause.pos, "Unreachable code", 2)
2199 2200 2201
            node.else_clause = None
        return node

2202

2203 2204 2205 2206
class YieldNodeCollector(TreeVisitor):

    def __init__(self):
        super(YieldNodeCollector, self).__init__()
2207
        self.yields = []
2208
        self.awaits = []
2209 2210
        self.returns = []
        self.has_return_value = False
2211

2212
    def visit_Node(self, node):
2213
        self.visitchildren(node)
2214 2215

    def visit_YieldExprNode(self, node):
2216
        self.yields.append(node)
Vitja Makarov's avatar
Vitja Makarov committed
2217
        self.visitchildren(node)
2218

2219 2220 2221 2222
    def visit_AwaitExprNode(self, node):
        self.awaits.append(node)
        self.visitchildren(node)

2223
    def visit_ReturnStatNode(self, node):
2224
        self.visitchildren(node)
2225 2226 2227
        if node.value:
            self.has_return_value = True
        self.returns.append(node)
2228 2229 2230 2231

    def visit_ClassDefNode(self, node):
        pass

2232
    def visit_FuncDefNode(self, node):
2233
        pass
2234

Vitja Makarov's avatar
Vitja Makarov committed
2235 2236 2237
    def visit_LambdaNode(self, node):
        pass

Vitja Makarov's avatar
Vitja Makarov committed
2238 2239 2240
    def visit_GeneratorExpressionNode(self, node):
        pass

2241

2242
class MarkClosureVisitor(CythonTransform):
2243 2244 2245 2246 2247 2248

    def visit_ModuleNode(self, node):
        self.needs_closure = False
        self.visitchildren(node)
        return node

Robert Bradshaw's avatar
Robert Bradshaw committed
2249 2250 2251 2252 2253
    def visit_FuncDefNode(self, node):
        self.needs_closure = False
        self.visitchildren(node)
        node.needs_closure = self.needs_closure
        self.needs_closure = True
2254

2255 2256 2257
        collector = YieldNodeCollector()
        collector.visitchildren(node)

2258 2259 2260 2261 2262 2263 2264 2265 2266 2267
        if node.is_async_def:
            if collector.yields:
                error(collector.yields[0].pos, "'yield' not allowed in async coroutines (use 'await')")
            yields = collector.awaits
        elif collector.yields:
            if collector.awaits:
                error(collector.yields[0].pos, "'await' not allowed in generators (use 'yield')")
            yields = collector.yields
        else:
            return node
2268

2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281
        for i, yield_expr in enumerate(yields, 1):
            yield_expr.label_num = i
        for retnode in collector.returns:
            retnode.in_generator = True

        gbody = Nodes.GeneratorBodyDefNode(
            pos=node.pos, name=node.name, body=node.body)
        coroutine = (Nodes.AsyncDefNode if node.is_async_def else Nodes.GeneratorDefNode)(
            pos=node.pos, name=node.name, args=node.args,
            star_arg=node.star_arg, starstar_arg=node.starstar_arg,
            doc=node.doc, decorators=node.decorators,
            gbody=gbody, lambda_name=node.lambda_name)
        return coroutine
2282

2283
    def visit_CFuncDefNode(self, node):
2284 2285 2286 2287
        self.needs_closure = False
        self.visitchildren(node)
        node.needs_closure = self.needs_closure
        self.needs_closure = True
2288 2289
        if node.needs_closure and node.overridable:
            error(node.pos, "closures inside cpdef functions not yet supported")
2290
        return node
Stefan Behnel's avatar
Stefan Behnel committed
2291 2292 2293 2294 2295 2296 2297 2298

    def visit_LambdaNode(self, node):
        self.needs_closure = False
        self.visitchildren(node)
        node.needs_closure = self.needs_closure
        self.needs_closure = True
        return node

Robert Bradshaw's avatar
Robert Bradshaw committed
2299 2300 2301 2302
    def visit_ClassDefNode(self, node):
        self.visitchildren(node)
        self.needs_closure = True
        return node
Stefan Behnel's avatar
Stefan Behnel committed
2303

2304

2305
class CreateClosureClasses(CythonTransform):
2306
    # Output closure classes in module scope for all functions
Vitja Makarov's avatar
Vitja Makarov committed
2307 2308 2309 2310 2311 2312 2313
    # that really need it.

    def __init__(self, context):
        super(CreateClosureClasses, self).__init__(context)
        self.path = []
        self.in_lambda = False

2314 2315 2316 2317 2318
    def visit_ModuleNode(self, node):
        self.module_scope = node.scope
        self.visitchildren(node)
        return node

Stefan Behnel's avatar
Stefan Behnel committed
2319
    def find_entries_used_in_closures(self, node):
Vitja Makarov's avatar
Vitja Makarov committed
2320 2321 2322 2323 2324
        from_closure = []
        in_closure = []
        for name, entry in node.local_scope.entries.items():
            if entry.from_closure:
                from_closure.append((name, entry))
Stefan Behnel's avatar
Stefan Behnel committed
2325
            elif entry.in_closure:
Vitja Makarov's avatar
Vitja Makarov committed
2326 2327 2328 2329
                in_closure.append((name, entry))
        return from_closure, in_closure

    def create_class_from_scope(self, node, target_module_scope, inner_node=None):
2330 2331 2332 2333 2334 2335
        # move local variables into closure
        if node.is_generator:
            for entry in node.local_scope.entries.values():
                if not entry.from_closure:
                    entry.in_closure = True

Stefan Behnel's avatar
Stefan Behnel committed
2336
        from_closure, in_closure = self.find_entries_used_in_closures(node)
Vitja Makarov's avatar
Vitja Makarov committed
2337 2338 2339 2340 2341 2342
        in_closure.sort()

        # Now from the begining
        node.needs_closure = False
        node.needs_outer_scope = False

2343
        func_scope = node.local_scope
Vitja Makarov's avatar
Vitja Makarov committed
2344 2345 2346 2347
        cscope = node.entry.scope
        while cscope.is_py_class_scope or cscope.is_c_class_scope:
            cscope = cscope.outer_scope

2348
        if not from_closure and (self.path or inner_node):
Vitja Makarov's avatar
Vitja Makarov committed
2349
            if not inner_node:
2350
                if not node.py_cfunc_node:
2351
                    raise InternalError("DefNode does not have assignment node")
2352
                inner_node = node.py_cfunc_node
Vitja Makarov's avatar
Vitja Makarov committed
2353 2354
            inner_node.needs_self_code = False
            node.needs_outer_scope = False
2355 2356

        if node.is_generator:
2357
            pass
2358
        elif not in_closure and not from_closure:
Vitja Makarov's avatar
Vitja Makarov committed
2359 2360 2361 2362 2363 2364 2365
            return
        elif not in_closure:
            func_scope.is_passthrough = True
            func_scope.scope_class = cscope.scope_class
            node.needs_outer_scope = True
            return

2366 2367 2368
        as_name = '%s_%s' % (
            target_module_scope.next_id(Naming.closure_class_prefix),
            node.entry.cname)
2369

Stefan Behnel's avatar
Stefan Behnel committed
2370 2371
        entry = target_module_scope.declare_c_class(
            name=as_name, pos=node.pos, defining=True,
2372
            implementing=True)
2373
        entry.type.is_final_type = True
Stefan Behnel's avatar
Stefan Behnel committed
2374

Robert Bradshaw's avatar
Robert Bradshaw committed
2375
        func_scope.scope_class = entry
2376
        class_scope = entry.type.scope
2377
        class_scope.is_internal = True
2378 2379
        if Options.closure_freelist_size:
            class_scope.directives['freelist'] = Options.closure_freelist_size
2380

Vitja Makarov's avatar
Vitja Makarov committed
2381 2382
        if from_closure:
            assert cscope.is_closure_scope
2383
            class_scope.declare_var(pos=node.pos,
Vitja Makarov's avatar
Vitja Makarov committed
2384
                                    name=Naming.outer_scope_cname,
2385
                                    cname=Naming.outer_scope_cname,
2386
                                    type=cscope.scope_class.type,
2387
                                    is_cdef=True)
Vitja Makarov's avatar
Vitja Makarov committed
2388 2389
            node.needs_outer_scope = True
        for name, entry in in_closure:
2390
            closure_entry = class_scope.declare_var(pos=entry.pos,
2391
                                    name=entry.name,
2392
                                    cname=entry.cname,
2393 2394
                                    type=entry.type,
                                    is_cdef=True)
2395 2396
            if entry.is_declared_generic:
                closure_entry.is_declared_generic = 1
Vitja Makarov's avatar
Vitja Makarov committed
2397 2398 2399 2400 2401
        node.needs_closure = True
        # Do it here because other classes are already checked
        target_module_scope.check_c_class(func_scope.scope_class)

    def visit_LambdaNode(self, node):
2402 2403 2404 2405
        if not isinstance(node.def_node, Nodes.DefNode):
            # fused function, an error has been previously issued
            return node

Vitja Makarov's avatar
Vitja Makarov committed
2406 2407 2408 2409 2410 2411 2412
        was_in_lambda = self.in_lambda
        self.in_lambda = True
        self.create_class_from_scope(node.def_node, self.module_scope, node)
        self.visitchildren(node)
        self.in_lambda = was_in_lambda
        return node

2413
    def visit_FuncDefNode(self, node):
Vitja Makarov's avatar
Vitja Makarov committed
2414 2415 2416 2417
        if self.in_lambda:
            self.visitchildren(node)
            return node
        if node.needs_closure or self.path:
Robert Bradshaw's avatar
Robert Bradshaw committed
2418
            self.create_class_from_scope(node, self.module_scope)
Vitja Makarov's avatar
Vitja Makarov committed
2419
            self.path.append(node)
2420
            self.visitchildren(node)
Vitja Makarov's avatar
Vitja Makarov committed
2421
            self.path.pop()
2422
        return node
2423

2424 2425 2426 2427
    def visit_GeneratorBodyDefNode(self, node):
        self.visitchildren(node)
        return node

2428
    def visit_CFuncDefNode(self, node):
Robert Bradshaw's avatar
Robert Bradshaw committed
2429 2430 2431 2432 2433
        if not node.overridable:
            return self.visit_FuncDefNode(node)
        else:
            self.visitchildren(node)
            return node
2434

2435 2436 2437 2438 2439 2440

class GilCheck(VisitorTransform):
    """
    Call `node.gil_check(env)` on each node to make sure we hold the
    GIL when we need it.  Raise an error when on Python operations
    inside a `nogil` environment.
2441 2442 2443

    Additionally, raise exceptions for closely nested with gil or with nogil
    statements. The latter would abort Python.
2444
    """
2445

2446 2447
    def __call__(self, root):
        self.env_stack = [root.scope]
2448
        self.nogil = False
2449 2450 2451 2452

        # True for 'cdef func() nogil:' functions, as the GIL may be held while
        # calling this function (thus contained 'nogil' blocks may be valid).
        self.nogil_declarator_only = False
2453 2454 2455 2456
        return super(GilCheck, self).__call__(root)

    def visit_FuncDefNode(self, node):
        self.env_stack.append(node.local_scope)
2457 2458
        was_nogil = self.nogil
        self.nogil = node.local_scope.nogil
Mark Florisson's avatar
Mark Florisson committed
2459

2460 2461 2462
        if self.nogil:
            self.nogil_declarator_only = True

2463 2464
        if self.nogil and node.nogil_check:
            node.nogil_check(node.local_scope)
Mark Florisson's avatar
Mark Florisson committed
2465

2466
        self.visitchildren(node)
2467 2468 2469 2470

        # This cannot be nested, so it doesn't need backup/restore
        self.nogil_declarator_only = False

2471
        self.env_stack.pop()
2472
        self.nogil = was_nogil
2473 2474 2475
        return node

    def visit_GILStatNode(self, node):
Mark Florisson's avatar
Mark Florisson committed
2476 2477 2478
        if self.nogil and node.nogil_check:
            node.nogil_check()

2479 2480
        was_nogil = self.nogil
        self.nogil = (node.state == 'nogil')
2481 2482 2483 2484 2485 2486 2487 2488 2489

        if was_nogil == self.nogil and not self.nogil_declarator_only:
            if not was_nogil:
                error(node.pos, "Trying to acquire the GIL while it is "
                                "already held.")
            else:
                error(node.pos, "Trying to release the GIL while it was "
                                "previously released.")

2490 2491 2492 2493 2494
        if isinstance(node.finally_clause, Nodes.StatListNode):
            # The finally clause of the GILStatNode is a GILExitNode,
            # which is wrapped in a StatListNode. Just unpack that.
            node.finally_clause, = node.finally_clause.stats

2495
        self.visitchildren(node)
2496
        self.nogil = was_nogil
2497 2498
        return node

Mark Florisson's avatar
Mark Florisson committed
2499
    def visit_ParallelRangeNode(self, node):
2500 2501
        if node.nogil:
            node.nogil = False
Mark Florisson's avatar
Mark Florisson committed
2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526
            node = Nodes.GILStatNode(node.pos, state='nogil', body=node)
            return self.visit_GILStatNode(node)

        if not self.nogil:
            error(node.pos, "prange() can only be used without the GIL")
            # Forget about any GIL-related errors that may occur in the body
            return None

        node.nogil_check(self.env_stack[-1])
        self.visitchildren(node)
        return node

    def visit_ParallelWithBlockNode(self, node):
        if not self.nogil:
            error(node.pos, "The parallel section may only be used without "
                            "the GIL")
            return None

        if node.nogil_check:
            # It does not currently implement this, but test for it anyway to
            # avoid potential future surprises
            node.nogil_check(self.env_stack[-1])

        self.visitchildren(node)
        return node
2527 2528 2529

    def visit_TryFinallyStatNode(self, node):
        """
2530
        Take care of try/finally statements in nogil code sections.
2531 2532 2533 2534 2535 2536
        """
        if not self.nogil or isinstance(node, Nodes.GILStatNode):
            return self.visit_Node(node)

        node.nogil_check = None
        node.is_try_finally_in_nogil = True
2537
        self.visitchildren(node)
Mark Florisson's avatar
Mark Florisson committed
2538
        return node
Mark Florisson's avatar
Mark Florisson committed
2539

2540
    def visit_Node(self, node):
2541 2542
        if self.env_stack and self.nogil and node.nogil_check:
            node.nogil_check(self.env_stack[-1])
2543
        self.visitchildren(node)
2544
        node.in_nogil_context = self.nogil
2545 2546
        return node

2547

Robert Bradshaw's avatar
Robert Bradshaw committed
2548
class TransformBuiltinMethods(EnvTransform):
2549 2550 2551
    """
    Replace Cython's own cython.* builtins by the corresponding tree nodes.
    """
Robert Bradshaw's avatar
Robert Bradshaw committed
2552

2553 2554 2555 2556 2557 2558
    def visit_SingleAssignmentNode(self, node):
        if node.declaration_only:
            return None
        else:
            self.visitchildren(node)
            return node
2559

2560
    def visit_AttributeNode(self, node):
2561
        self.visitchildren(node)
2562 2563 2564 2565
        return self.visit_cython_attribute(node)

    def visit_NameNode(self, node):
        return self.visit_cython_attribute(node)
2566

2567 2568
    def visit_cython_attribute(self, node):
        attribute = node.as_cython_attribute()
2569 2570
        if attribute:
            if attribute == u'compiled':
2571
                node = ExprNodes.BoolNode(node.pos, value=True)
Stefan Behnel's avatar
Stefan Behnel committed
2572
            elif attribute == u'__version__':
2573 2574
                from .. import __version__ as version
                node = ExprNodes.StringNode(node.pos, value=EncodedString(version))
2575
            elif attribute == u'NULL':
2576
                node = ExprNodes.NullNode(node.pos)
2577
            elif attribute in (u'set', u'frozenset', u'staticmethod'):
2578 2579
                node = ExprNodes.NameNode(node.pos, name=EncodedString(attribute),
                                          entry=self.current_env().builtin_scope().lookup_here(attribute))
2580 2581
            elif PyrexTypes.parse_basic_type(attribute):
                pass
2582
            elif self.context.cython_scope.lookup_qualified_name(attribute):
2583 2584
                pass
            else:
Robert Bradshaw's avatar
Robert Bradshaw committed
2585
                error(node.pos, u"'%s' not a valid cython attribute or is being used incorrectly" % attribute)
2586 2587
        return node

Vitja Makarov's avatar
Vitja Makarov committed
2588 2589 2590 2591 2592 2593 2594 2595 2596 2597 2598
    def visit_ExecStatNode(self, node):
        lenv = self.current_env()
        self.visitchildren(node)
        if len(node.args) == 1:
            node.args.append(ExprNodes.GlobalsExprNode(node.pos))
            if not lenv.is_module_scope:
                node.args.append(
                    ExprNodes.LocalsExprNode(
                        node.pos, self.current_scope_node(), lenv))
        return node

2599
    def _inject_locals(self, node, func_name):
2600
        # locals()/dir()/vars() builtins
2601 2602 2603 2604 2605 2606
        lenv = self.current_env()
        entry = lenv.lookup_here(func_name)
        if entry:
            # not the builtin
            return node
        pos = node.pos
2607 2608
        if func_name in ('locals', 'vars'):
            if func_name == 'locals' and len(node.args) > 0:
2609 2610 2611
                error(self.pos, "Builtin 'locals()' called with wrong number of args, expected 0, got %d"
                      % len(node.args))
                return node
2612 2613 2614 2615 2616 2617
            elif func_name == 'vars':
                if len(node.args) > 1:
                    error(self.pos, "Builtin 'vars()' called with wrong number of args, expected 0-1, got %d"
                          % len(node.args))
                if len(node.args) > 0:
                    return node # nothing to do
2618
            return ExprNodes.LocalsExprNode(pos, self.current_scope_node(), lenv)
2619
        else: # dir()
2620 2621 2622
            if len(node.args) > 1:
                error(self.pos, "Builtin 'dir()' called with wrong number of args, expected 0-1, got %d"
                      % len(node.args))
2623
            if len(node.args) > 0:
2624 2625
                # optimised in Builtin.py
                return node
2626 2627 2628 2629 2630 2631
            if lenv.is_py_class_scope or lenv.is_module_scope:
                if lenv.is_py_class_scope:
                    pyclass = self.current_scope_node()
                    locals_dict = ExprNodes.CloneNode(pyclass.dict)
                else:
                    locals_dict = ExprNodes.GlobalsExprNode(pos)
2632
                return ExprNodes.SortedDictKeysNode(locals_dict)
2633 2634 2635
            local_names = sorted(var.name for var in lenv.entries.values() if var.name)
            items = [ExprNodes.IdentifierStringNode(pos, value=var)
                     for var in local_names]
2636
            return ExprNodes.ListNode(pos, args=items)
2637

2638 2639 2640 2641 2642 2643 2644 2645 2646 2647 2648 2649 2650 2651
    def visit_PrimaryCmpNode(self, node):
        # special case: for in/not-in test, we do not need to sort locals()
        self.visitchildren(node)
        if node.operator in 'not_in':  # in/not_in
            if isinstance(node.operand2, ExprNodes.SortedDictKeysNode):
                arg = node.operand2.arg
                if isinstance(arg, ExprNodes.NoneCheckNode):
                    arg = arg.arg
                node.operand2 = arg
        return node

    def visit_CascadedCmpNode(self, node):
        return self.visit_PrimaryCmpNode(node)

2652 2653 2654 2655 2656 2657 2658 2659 2660 2661 2662 2663
    def _inject_eval(self, node, func_name):
        lenv = self.current_env()
        entry = lenv.lookup_here(func_name)
        if entry or len(node.args) != 1:
            return node
        # Inject globals and locals
        node.args.append(ExprNodes.GlobalsExprNode(node.pos))
        if not lenv.is_module_scope:
            node.args.append(
                ExprNodes.LocalsExprNode(
                    node.pos, self.current_scope_node(), lenv))
        return node
2664

2665 2666 2667 2668 2669 2670 2671
    def _inject_super(self, node, func_name):
        lenv = self.current_env()
        entry = lenv.lookup_here(func_name)
        if entry or node.args:
            return node
        # Inject no-args super
        def_node = self.current_scope_node()
2672
        if (not isinstance(def_node, Nodes.DefNode) or not def_node.args or
2673 2674 2675
            len(self.env_stack) < 2):
            return node
        class_node, class_scope = self.env_stack[-2]
2676
        if class_scope.is_py_class_scope:
2677 2678 2679 2680 2681 2682 2683
            def_node.requires_classobj = True
            class_node.class_cell.is_active = True
            node.args = [
                ExprNodes.ClassCellNode(
                    node.pos, is_generator=def_node.is_generator),
                ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
                ]
2684 2685 2686 2687 2688 2689 2690
        elif class_scope.is_c_class_scope:
            node.args = [
                ExprNodes.NameNode(
                    node.pos, name=class_node.scope.name,
                    entry=class_node.entry),
                ExprNodes.NameNode(node.pos, name=def_node.args[0].name)
                ]
2691 2692
        return node

2693
    def visit_SimpleCallNode(self, node):
2694
        # cython.foo
2695
        function = node.function.as_cython_attribute()
2696
        if function:
2697 2698 2699 2700
            if function in InterpretCompilerDirectives.unop_method_nodes:
                if len(node.args) != 1:
                    error(node.function.pos, u"%s() takes exactly one argument" % function)
                else:
Stefan Behnel's avatar
Stefan Behnel committed
2701 2702
                    node = InterpretCompilerDirectives.unop_method_nodes[function](
                        node.function.pos, operand=node.args[0])
Robert Bradshaw's avatar
Robert Bradshaw committed
2703 2704 2705 2706
            elif function in InterpretCompilerDirectives.binop_method_nodes:
                if len(node.args) != 2:
                    error(node.function.pos, u"%s() takes exactly two arguments" % function)
                else:
Stefan Behnel's avatar
Stefan Behnel committed
2707 2708
                    node = InterpretCompilerDirectives.binop_method_nodes[function](
                        node.function.pos, operand1=node.args[0], operand2=node.args[1])
2709
            elif function == u'cast':
2710
                if len(node.args) != 2:
2711
                    error(node.function.pos, u"cast() takes exactly two arguments")
2712
                else:
Stefan Behnel's avatar
Stefan Behnel committed
2713
                    type = node.args[0].analyse_as_type(self.current_env())
2714
                    if type:
2715
                        node = ExprNodes.TypecastNode(node.function.pos, type=type, operand=node.args[1])
2716 2717 2718 2719
                    else:
                        error(node.args[0].pos, "Not a type")
            elif function == u'sizeof':
                if len(node.args) != 1:
Robert Bradshaw's avatar
Robert Bradshaw committed
2720
                    error(node.function.pos, u"sizeof() takes exactly one argument")
2721
                else:
Stefan Behnel's avatar
Stefan Behnel committed
2722
                    type = node.args[0].analyse_as_type(self.current_env())
2723
                    if type:
2724
                        node = ExprNodes.SizeofTypeNode(node.function.pos, arg_type=type)
2725
                    else:
2726
                        node = ExprNodes.SizeofVarNode(node.function.pos, operand=node.args[0])
2727 2728
            elif function == 'cmod':
                if len(node.args) != 2:
Robert Bradshaw's avatar
Robert Bradshaw committed
2729
                    error(node.function.pos, u"cmod() takes exactly two arguments")
2730
                else:
2731
                    node = ExprNodes.binop_node(node.function.pos, '%', node.args[0], node.args[1])
2732 2733 2734
                    node.cdivision = True
            elif function == 'cdiv':
                if len(node.args) != 2:
Robert Bradshaw's avatar
Robert Bradshaw committed
2735
                    error(node.function.pos, u"cdiv() takes exactly two arguments")
2736
                else:
2737
                    node = ExprNodes.binop_node(node.function.pos, '/', node.args[0], node.args[1])
2738
                    node.cdivision = True
2739
            elif function == u'set':
2740
                node.function = ExprNodes.NameNode(node.pos, name=EncodedString('set'))
Robert Bradshaw's avatar
Robert Bradshaw committed
2741 2742
            elif function == u'staticmethod':
                node.function = ExprNodes.NameNode(node.pos, name=EncodedString('staticmethod'))
2743 2744
            elif self.context.cython_scope.lookup_qualified_name(function):
                pass
2745
            else:
2746 2747
                error(node.function.pos,
                      u"'%s' not a valid cython language construct" % function)
2748

2749
        self.visitchildren(node)
2750 2751 2752 2753 2754

        if isinstance(node, ExprNodes.SimpleCallNode) and node.function.is_name:
            func_name = node.function.name
            if func_name in ('dir', 'locals', 'vars'):
                return self._inject_locals(node, func_name)
2755 2756
            if func_name == 'eval':
                return self._inject_eval(node, func_name)
2757 2758
            if func_name == 'super':
                return self._inject_super(node, func_name)
Robert Bradshaw's avatar
Robert Bradshaw committed
2759
        return node
2760 2761


2762 2763 2764 2765 2766 2767 2768 2769 2770 2771 2772 2773 2774 2775
class ReplaceFusedTypeChecks(VisitorTransform):
    """
    This is not a transform in the pipeline. It is invoked on the specific
    versions of a cdef function with fused argument types. It filters out any
    type branches that don't match. e.g.

        if fused_t is mytype:
            ...
        elif fused_t in other_fused_type:
            ...
    """
    def __init__(self, local_scope):
        super(ReplaceFusedTypeChecks, self).__init__()
        self.local_scope = local_scope
Stefan Behnel's avatar
Stefan Behnel committed
2776
        # defer the import until now to avoid circular import time dependencies
2777 2778
        from .Optimize import ConstantFolding
        self.transform = ConstantFolding(reevaluate=True)
2779 2780

    def visit_IfStatNode(self, node):
2781 2782 2783 2784
        """
        Filters out any if clauses with false compile time type check
        expression.
        """
2785
        self.visitchildren(node)
2786
        return self.transform(node)
2787

2788 2789 2790 2791 2792
    def visit_PrimaryCmpNode(self, node):
        type1 = node.operand1.analyse_as_type(self.local_scope)
        type2 = node.operand2.analyse_as_type(self.local_scope)

        if type1 and type2:
Mark Florisson's avatar
Mark Florisson committed
2793 2794
            false_node = ExprNodes.BoolNode(node.pos, value=False)
            true_node = ExprNodes.BoolNode(node.pos, value=True)
2795 2796 2797 2798

            type1 = self.specialize_type(type1, node.operand1.pos)
            op = node.operator

2799
            if op in ('is', 'is_not', '==', '!='):
2800 2801 2802 2803 2804 2805
                type2 = self.specialize_type(type2, node.operand2.pos)

                is_same = type1.same_as(type2)
                eq = op in ('is', '==')

                if (is_same and eq) or (not is_same and not eq):
Mark Florisson's avatar
Mark Florisson committed
2806
                    return true_node
2807 2808 2809 2810 2811 2812 2813 2814 2815 2816 2817 2818 2819 2820

            elif op in ('in', 'not_in'):
                # We have to do an instance check directly, as operand2
                # needs to be a fused type and not a type with a subtype
                # that is fused. First unpack the typedef
                if isinstance(type2, PyrexTypes.CTypedefType):
                    type2 = type2.typedef_base_type

                if type1.is_fused:
                    error(node.operand1.pos, "Type is fused")
                elif not type2.is_fused:
                    error(node.operand2.pos,
                          "Can only use 'in' or 'not in' on a fused type")
                else:
2821
                    types = PyrexTypes.get_specialized_types(type2)
2822

2823 2824
                    for specialized_type in types:
                        if type1.same_as(specialized_type):
2825
                            if op == 'in':
Mark Florisson's avatar
Mark Florisson committed
2826
                                return true_node
2827
                            else:
Mark Florisson's avatar
Mark Florisson committed
2828
                                return false_node
2829 2830

                    if op == 'not_in':
Mark Florisson's avatar
Mark Florisson committed
2831
                        return true_node
2832

Mark Florisson's avatar
Mark Florisson committed
2833
            return false_node
2834 2835 2836 2837 2838 2839 2840 2841 2842 2843 2844 2845 2846 2847 2848

        return node

    def specialize_type(self, type, pos):
        try:
            return type.specialize(self.local_scope.fused_to_specific)
        except KeyError:
            error(pos, "Type is not specific")
            return type

    def visit_Node(self, node):
        self.visitchildren(node)
        return node


Mark Florisson's avatar
Mark Florisson committed
2849
class DebugTransform(CythonTransform):
2850
    """
Mark Florisson's avatar
Mark Florisson committed
2851
    Write debug information for this Cython module.
2852
    """
2853

2854
    def __init__(self, context, options, result):
Mark Florisson's avatar
Mark Florisson committed
2855
        super(DebugTransform, self).__init__(context)
Robert Bradshaw's avatar
Robert Bradshaw committed
2856
        self.visited = set()
2857
        # our treebuilder and debug output writer
Mark Florisson's avatar
Mark Florisson committed
2858
        # (see Cython.Debugger.debug_output.CythonDebugWriter)
2859
        self.tb = self.context.gdb_debug_outputwriter
2860
        #self.c_output_file = options.output_file
2861
        self.c_output_file = result.c_file
2862

2863 2864 2865
        # Closure support, basically treat nested functions as if the AST were
        # never nested
        self.nested_funcdefs = []
2866

Mark Florisson's avatar
Mark Florisson committed
2867 2868
        # tells visit_NameNode whether it should register step-into functions
        self.register_stepinto = False
2869

2870
    def visit_ModuleNode(self, node):
Mark Florisson's avatar
Mark Florisson committed
2871
        self.tb.module_name = node.full_module_name
2872
        attrs = dict(
Mark Florisson's avatar
Mark Florisson committed
2873
            module_name=node.full_module_name,
Mark Florisson's avatar
Mark Florisson committed
2874 2875
            filename=node.pos[0].filename,
            c_filename=self.c_output_file)
2876

2877
        self.tb.start('Module', attrs)
2878

2879
        # serialize functions
Mark Florisson's avatar
Mark Florisson committed
2880
        self.tb.start('Functions')
2881
        # First, serialize functions normally...
2882
        self.visitchildren(node)
2883

2884 2885 2886
        # ... then, serialize nested functions
        for nested_funcdef in self.nested_funcdefs:
            self.visit_FuncDefNode(nested_funcdef)
2887

2888 2889 2890
        self.register_stepinto = True
        self.serialize_modulenode_as_function(node)
        self.register_stepinto = False
2891
        self.tb.end('Functions')
2892

2893
        # 2.3 compatibility. Serialize global variables
Mark Florisson's avatar
Mark Florisson committed
2894
        self.tb.start('Globals')
2895
        entries = {}
Mark Florisson's avatar
Mark Florisson committed
2896

2897
        for k, v in node.scope.entries.iteritems():
Mark Florisson's avatar
Mark Florisson committed
2898
            if (v.qualified_name not in self.visited and not
2899
                v.name.startswith('__pyx_') and not
Mark Florisson's avatar
Mark Florisson committed
2900 2901
                v.type.is_cfunction and not
                v.type.is_extension_type):
2902
                entries[k]= v
2903

2904 2905
        self.serialize_local_variables(entries)
        self.tb.end('Globals')
Mark Florisson's avatar
Mark Florisson committed
2906 2907
        # self.tb.end('Module') # end Module after the line number mapping in
        # Cython.Compiler.ModuleNode.ModuleNode._serialize_lineno_map
2908
        return node
2909 2910

    def visit_FuncDefNode(self, node):
2911
        self.visited.add(node.local_scope.qualified_name)
2912 2913 2914 2915 2916 2917 2918 2919

        if getattr(node, 'is_wrapper', False):
            return node

        if self.register_stepinto:
            self.nested_funcdefs.append(node)
            return node

2920
        # node.entry.visibility = 'extern'
2921 2922 2923 2924
        if node.py_func is None:
            pf_cname = ''
        else:
            pf_cname = node.py_func.entry.func_cname
2925

2926
        attrs = dict(
2927
            name=node.entry.name or getattr(node, 'name', '<unknown>'),
2928 2929 2930 2931
            cname=node.entry.func_cname,
            pf_cname=pf_cname,
            qualified_name=node.local_scope.qualified_name,
            lineno=str(node.pos[1]))
2932

2933
        self.tb.start('Function', attrs=attrs)
2934

Mark Florisson's avatar
Mark Florisson committed
2935
        self.tb.start('Locals')
2936 2937
        self.serialize_local_variables(node.local_scope.entries)
        self.tb.end('Locals')
Mark Florisson's avatar
Mark Florisson committed
2938 2939

        self.tb.start('Arguments')
2940
        for arg in node.local_scope.arg_entries:
Mark Florisson's avatar
Mark Florisson committed
2941 2942
            self.tb.start(arg.name)
            self.tb.end(arg.name)
2943
        self.tb.end('Arguments')
Mark Florisson's avatar
Mark Florisson committed
2944 2945

        self.tb.start('StepIntoFunctions')
Mark Florisson's avatar
Mark Florisson committed
2946
        self.register_stepinto = True
Mark Florisson's avatar
Mark Florisson committed
2947
        self.visitchildren(node)
Mark Florisson's avatar
Mark Florisson committed
2948
        self.register_stepinto = False
Mark Florisson's avatar
Mark Florisson committed
2949
        self.tb.end('StepIntoFunctions')
2950
        self.tb.end('Function')
Mark Florisson's avatar
Mark Florisson committed
2951 2952 2953 2954

        return node

    def visit_NameNode(self, node):
2955
        if (self.register_stepinto and
2956
            node.type is not None and
2957
            node.type.is_cfunction and
2958 2959
            getattr(node, 'is_called', False) and
            node.entry.func_cname is not None):
2960 2961 2962 2963
            # don't check node.entry.in_cinclude, as 'cdef extern: ...'
            # declared functions are not 'in_cinclude'.
            # This means we will list called 'cdef' functions as
            # "step into functions", but this is not an issue as they will be
Mark Florisson's avatar
Mark Florisson committed
2964
            # recognized as Cython functions anyway.
Mark Florisson's avatar
Mark Florisson committed
2965 2966 2967
            attrs = dict(name=node.entry.func_cname)
            self.tb.start('StepIntoFunction', attrs=attrs)
            self.tb.end('StepIntoFunction')
2968

Mark Florisson's avatar
Mark Florisson committed
2969
        self.visitchildren(node)
2970
        return node
2971

2972 2973 2974 2975 2976 2977 2978
    def serialize_modulenode_as_function(self, node):
        """
        Serialize the module-level code as a function so the debugger will know
        it's a "relevant frame" and it will know where to set the breakpoint
        for 'break modulename'.
        """
        name = node.full_module_name.rpartition('.')[-1]
2979

2980 2981
        cname_py2 = 'init' + name
        cname_py3 = 'PyInit_' + name
2982

2983 2984 2985 2986
        py2_attrs = dict(
            name=name,
            cname=cname_py2,
            pf_cname='',
2987
            # Ignore the qualified_name, breakpoints should be set using
2988 2989 2990 2991 2992
            # `cy break modulename:lineno` for module-level breakpoints.
            qualified_name='',
            lineno='1',
            is_initmodule_function="True",
        )
2993

2994
        py3_attrs = dict(py2_attrs, cname=cname_py3)
2995

2996 2997
        self._serialize_modulenode_as_function(node, py2_attrs)
        self._serialize_modulenode_as_function(node, py3_attrs)
2998

2999 3000
    def _serialize_modulenode_as_function(self, node, attrs):
        self.tb.start('Function', attrs=attrs)
3001

3002 3003 3004 3005 3006 3007 3008 3009 3010 3011 3012 3013
        self.tb.start('Locals')
        self.serialize_local_variables(node.scope.entries)
        self.tb.end('Locals')

        self.tb.start('Arguments')
        self.tb.end('Arguments')

        self.tb.start('StepIntoFunctions')
        self.register_stepinto = True
        self.visitchildren(node)
        self.register_stepinto = False
        self.tb.end('StepIntoFunctions')
3014

3015
        self.tb.end('Function')
3016

3017 3018
    def serialize_local_variables(self, entries):
        for entry in entries.values():
3019 3020 3021
            if not entry.cname:
                # not a local variable
                continue
3022
            if entry.type.is_pyobject:
Mark Florisson's avatar
Mark Florisson committed
3023
                vartype = 'PythonObject'
3024 3025
            else:
                vartype = 'CObject'
3026

3027 3028 3029
            if entry.from_closure:
                # We're dealing with a closure where a variable from an outer
                # scope is accessed, get it from the scope object.
3030
                cname = '%s->%s' % (Naming.cur_scope_cname,
3031
                                    entry.outer_entry.cname)
3032

3033
                qname = '%s.%s.%s' % (entry.scope.outer_scope.qualified_name,
3034
                                      entry.scope.name,
3035
                                      entry.name)
3036
            elif entry.in_closure:
3037
                cname = '%s->%s' % (Naming.cur_scope_cname,
3038 3039
                                    entry.cname)
                qname = entry.qualified_name
3040 3041 3042
            else:
                cname = entry.cname
                qname = entry.qualified_name
3043

3044 3045 3046 3047 3048 3049 3050
            if not entry.pos:
                # this happens for variables that are not in the user's code,
                # e.g. for the global __builtins__, __doc__, etc. We can just
                # set the lineno to 0 for those.
                lineno = '0'
            else:
                lineno = str(entry.pos[1])
3051

3052 3053 3054
            attrs = dict(
                name=entry.name,
                cname=cname,
3055
                qualified_name=qname,
3056 3057
                type=vartype,
                lineno=lineno)
3058

Mark Florisson's avatar
Mark Florisson committed
3059 3060
            self.tb.start('LocalVar', attrs)
            self.tb.end('LocalVar')