ExprNodes.py 377 KB
Newer Older
William Stein's avatar
William Stein committed
1
#
2
#   Parse tree nodes for expressions
William Stein's avatar
William Stein committed
3 4
#

5 6
import cython
cython.declare(error=object, warning=object, warn_once=object, InternalError=object,
7 8
               CompileError=object, UtilityCode=object, TempitaUtilityCode=object,
               StringEncoding=object, operator=object,
9
               Naming=object, Nodes=object, PyrexTypes=object, py_object_type=object,
Stefan Behnel's avatar
Stefan Behnel committed
10
               list_type=object, tuple_type=object, set_type=object, dict_type=object,
11 12 13 14
               unicode_type=object, str_type=object, bytes_type=object, type_type=object,
               Builtin=object, Symtab=object, Utils=object, find_coercion_error=object,
               debug_disposal_code=object, debug_temp_alloc=object, debug_coercion=object)

15
import sys
16
import copy
17
import operator
William Stein's avatar
William Stein committed
18

19
from Errors import error, warning, warn_once, InternalError, CompileError
20
from Errors import hold_errors, release_errors, held_errors, report_error
21
from Code import UtilityCode, TempitaUtilityCode
22
import StringEncoding
William Stein's avatar
William Stein committed
23
import Naming
Robert Bradshaw's avatar
Robert Bradshaw committed
24
import Nodes
William Stein's avatar
William Stein committed
25 26
from Nodes import Node
import PyrexTypes
27
from PyrexTypes import py_object_type, c_long_type, typecast, error_type, \
28
     unspecified_type, cython_memoryview_ptr_type
29
import TypeSlots
30 31
from Builtin import list_type, tuple_type, set_type, dict_type, \
     unicode_type, str_type, bytes_type, type_type
32
import Builtin
William Stein's avatar
William Stein committed
33 34
import Symtab
import Options
35
from Cython import Utils
36
from Annotate import AnnotationItem
37
from Cython.Compiler import Future
William Stein's avatar
William Stein committed
38
from Cython.Debugging import print_call_chain
William Stein's avatar
William Stein committed
39 40 41
from DebugFlags import debug_disposal_code, debug_temp_alloc, \
    debug_coercion

42 43 44 45 46
try:
    from __builtin__ import basestring
except ImportError:
    basestring = str # Python 3

Stefan Behnel's avatar
Stefan Behnel committed
47
class NotConstant(object):
48 49 50 51 52 53 54 55
    _obj = None

    def __new__(cls):
        if NotConstant._obj is None:
            NotConstant._obj = super(NotConstant, cls).__new__(cls)

        return NotConstant._obj

Stefan Behnel's avatar
Stefan Behnel committed
56 57 58
    def __repr__(self):
        return "<NOT CONSTANT>"

59
not_a_constant = NotConstant()
60
constant_value_not_set = object()
61

62 63 64 65 66 67 68 69 70 71
# error messages when coercing from key[0] to key[1]
find_coercion_error = {
    # string related errors
    (Builtin.unicode_type, Builtin.bytes_type) : "Cannot convert Unicode string to 'bytes' implicitly, encoding required.",
    (Builtin.unicode_type, Builtin.str_type)   : "Cannot convert Unicode string to 'str' implicitly. This is not portable and requires explicit encoding.",
    (Builtin.unicode_type, PyrexTypes.c_char_ptr_type) : "Unicode objects do not support coercion to C types.",
    (Builtin.bytes_type, Builtin.unicode_type) : "Cannot convert 'bytes' object to unicode implicitly, decoding required",
    (Builtin.bytes_type, Builtin.str_type) : "Cannot convert 'bytes' object to str implicitly. This is not portable to Py3.",
    (Builtin.str_type, Builtin.unicode_type) : "str objects do not support coercion to unicode, use a unicode string literal instead (u'')",
    (Builtin.str_type, Builtin.bytes_type) : "Cannot convert 'str' to 'bytes' implicitly. This is not portable.",
72
    (Builtin.str_type, PyrexTypes.c_char_ptr_type) : "'str' objects do not support coercion to C types (use 'bytes'?).",
73 74 75 76 77
    (PyrexTypes.c_char_ptr_type, Builtin.unicode_type) : "Cannot convert 'char*' to unicode implicitly, decoding required",
    (PyrexTypes.c_uchar_ptr_type, Builtin.unicode_type) : "Cannot convert 'char*' to unicode implicitly, decoding required",
    }.get


William Stein's avatar
William Stein committed
78 79 80 81 82 83
class ExprNode(Node):
    #  subexprs     [string]     Class var holding names of subexpr node attrs
    #  type         PyrexType    Type of the result
    #  result_code  string       Code fragment
    #  result_ctype string       C type of result_code if different from type
    #  is_temp      boolean      Result is in a temporary variable
84
    #  is_sequence_constructor
William Stein's avatar
William Stein committed
85
    #               boolean      Is a list or tuple constructor expression
86
    #  is_starred   boolean      Is a starred expression (e.g. '*a')
William Stein's avatar
William Stein committed
87 88 89
    #  saved_subexpr_nodes
    #               [ExprNode or [ExprNode or None] or None]
    #                            Cached result of subexpr_nodes()
90
    #  use_managed_ref boolean   use ref-counted temps/assignments/etc.
91 92 93
    #  result_is_used  boolean   indicates that the result will be dropped and the
    #                            result_code/temp_result can safely be set to None

William Stein's avatar
William Stein committed
94
    result_ctype = None
95
    type = None
96 97
    temp_code = None
    old_temp = None # error checker for multiple frees etc.
98
    use_managed_ref = True # can be set by optimisation transforms
99
    result_is_used = True
William Stein's avatar
William Stein committed
100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126

    #  The Analyse Expressions phase for expressions is split
    #  into two sub-phases:
    #
    #    Analyse Types
    #      Determines the result type of the expression based
    #      on the types of its sub-expressions, and inserts
    #      coercion nodes into the expression tree where needed.
    #      Marks nodes which will need to have temporary variables
    #      allocated.
    #
    #    Allocate Temps
    #      Allocates temporary variables where needed, and fills
    #      in the result_code field of each node.
    #
    #  ExprNode provides some convenience routines which
    #  perform both of the above phases. These should only
    #  be called from statement nodes, and only when no
    #  coercion nodes need to be added around the expression
    #  being analysed. In that case, the above two phases
    #  should be invoked separately.
    #
    #  Framework code in ExprNode provides much of the common
    #  processing for the various phases. It makes use of the
    #  'subexprs' class attribute of ExprNodes, which should
    #  contain a list of the names of attributes which can
    #  hold sub-nodes or sequences of sub-nodes.
127 128
    #
    #  The framework makes use of a number of abstract methods.
William Stein's avatar
William Stein committed
129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150
    #  Their responsibilities are as follows.
    #
    #    Declaration Analysis phase
    #
    #      analyse_target_declaration
    #        Called during the Analyse Declarations phase to analyse
    #        the LHS of an assignment or argument of a del statement.
    #        Nodes which cannot be the LHS of an assignment need not
    #        implement it.
    #
    #    Expression Analysis phase
    #
    #      analyse_types
    #        - Call analyse_types on all sub-expressions.
    #        - Check operand types, and wrap coercion nodes around
    #          sub-expressions where needed.
    #        - Set the type of this node.
    #        - If a temporary variable will be required for the
    #          result, set the is_temp flag of this node.
    #
    #      analyse_target_types
    #        Called during the Analyse Types phase to analyse
151
    #        the LHS of an assignment or argument of a del
William Stein's avatar
William Stein committed
152 153
    #        statement. Similar responsibilities to analyse_types.
    #
154 155 156 157
    #      target_code
    #        Called by the default implementation of allocate_target_temps.
    #        Should return a C lvalue for assigning to the node. The default
    #        implementation calls calculate_result_code.
William Stein's avatar
William Stein committed
158 159 160 161
    #
    #      check_const
    #        - Check that this node and its subnodes form a
    #          legal constant expression. If so, do nothing,
162
    #          otherwise call not_const.
William Stein's avatar
William Stein committed
163
    #
164
    #        The default implementation of check_const
William Stein's avatar
William Stein committed
165 166 167 168 169 170 171 172
    #        assumes that the expression is not constant.
    #
    #      check_const_addr
    #        - Same as check_const, except check that the
    #          expression is a C lvalue whose address is
    #          constant. Otherwise, call addr_not_const.
    #
    #        The default implementation of calc_const_addr
173
    #        assumes that the expression is not a constant
William Stein's avatar
William Stein committed
174 175 176 177 178 179 180 181 182 183 184 185
    #        lvalue.
    #
    #   Code Generation phase
    #
    #      generate_evaluation_code
    #        - Call generate_evaluation_code for sub-expressions.
    #        - Perform the functions of generate_result_code
    #          (see below).
    #        - If result is temporary, call generate_disposal_code
    #          on all sub-expressions.
    #
    #        A default implementation of generate_evaluation_code
186
    #        is provided which uses the following abstract methods:
William Stein's avatar
William Stein committed
187 188 189 190 191 192
    #
    #          generate_result_code
    #            - Generate any C statements necessary to calculate
    #              the result of this node from the results of its
    #              sub-expressions.
    #
193
    #          calculate_result_code
194 195
    #            - Should return a C code fragment evaluating to the
    #              result. This is only called when the result is not
196 197
    #              a temporary.
    #
William Stein's avatar
William Stein committed
198 199 200 201 202 203 204 205 206 207 208 209 210 211 212
    #      generate_assignment_code
    #        Called on the LHS of an assignment.
    #        - Call generate_evaluation_code for sub-expressions.
    #        - Generate code to perform the assignment.
    #        - If the assignment absorbed a reference, call
    #          generate_post_assignment_code on the RHS,
    #          otherwise call generate_disposal_code on it.
    #
    #      generate_deletion_code
    #        Called on an argument of a del statement.
    #        - Call generate_evaluation_code for sub-expressions.
    #        - Generate code to perform the deletion.
    #        - Call generate_disposal_code on all sub-expressions.
    #
    #
213

William Stein's avatar
William Stein committed
214
    is_sequence_constructor = 0
215
    is_string_literal = 0
William Stein's avatar
William Stein committed
216
    is_attribute = 0
217

William Stein's avatar
William Stein committed
218 219
    saved_subexpr_nodes = None
    is_temp = 0
220
    is_target = 0
221
    is_starred = 0
William Stein's avatar
William Stein committed
222

223 224
    constant_result = constant_value_not_set

225 226 227
    # whether this node with a memoryview type should be broadcast
    memslice_broadcast = False

228 229 230 231
    try:
        _get_child_attrs = operator.attrgetter('subexprs')
    except AttributeError:
        # Python 2.3
232
        def __get_child_attrs(self):
233
            return self.subexprs
234
        _get_child_attrs = __get_child_attrs
235
    child_attrs = property(fget=_get_child_attrs)
236

William Stein's avatar
William Stein committed
237 238 239 240
    def not_implemented(self, method_name):
        print_call_chain(method_name, "not implemented") ###
        raise InternalError(
            "%s.%s not implemented" %
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
241
                (self.__class__.__name__, method_name))
242

William Stein's avatar
William Stein committed
243 244
    def is_lvalue(self):
        return 0
245

246
    def is_addressable(self):
247
        return self.is_lvalue() and not self.type.is_memoryviewslice
248

William Stein's avatar
William Stein committed
249 250 251 252 253 254 255 256 257 258 259
    def is_ephemeral(self):
        #  An ephemeral node is one whose result is in
        #  a Python temporary and we suspect there are no
        #  other references to it. Certain operations are
        #  disallowed on such values, since they are
        #  likely to result in a dangling pointer.
        return self.type.is_pyobject and self.is_temp

    def subexpr_nodes(self):
        #  Extract a list of subexpression nodes based
        #  on the contents of the subexprs class attribute.
260 261 262
        nodes = []
        for name in self.subexprs:
            item = getattr(self, name)
Stefan Behnel's avatar
Stefan Behnel committed
263 264
            if item is not None:
                if type(item) is list:
265
                    nodes.extend(item)
Stefan Behnel's avatar
Stefan Behnel committed
266 267
                else:
                    nodes.append(item)
268
        return nodes
269

270
    def result(self):
271 272 273
        if self.is_temp:
            return self.temp_code
        else:
274
            return self.calculate_result_code()
275

William Stein's avatar
William Stein committed
276 277
    def result_as(self, type = None):
        #  Return the result code cast to the specified C type.
278
        return typecast(type, self.ctype(), self.result())
279

William Stein's avatar
William Stein committed
280 281 282
    def py_result(self):
        #  Return the result code cast to PyObject *.
        return self.result_as(py_object_type)
283

William Stein's avatar
William Stein committed
284 285 286 287
    def ctype(self):
        #  Return the native C type of the result (i.e. the
        #  C type of the result_code expression).
        return self.result_ctype or self.type
288

289
    def get_constant_c_result_code(self):
290
        # Return the constant value of this node as a result code
291 292 293 294 295 296 297
        # string, or None if the node is not constant.  This method
        # can be called when the constant result code is required
        # before the code generation phase.
        #
        # The return value is a string that can represent a simple C
        # value, a constant C name or a constant C expression.  If the
        # node type depends on Python code, this must return None.
298 299
        return None

300
    def calculate_constant_result(self):
301 302 303 304 305
        # Calculate the constant compile time result value of this
        # expression and store it in ``self.constant_result``.  Does
        # nothing by default, thus leaving ``self.constant_result``
        # unknown.  If valid, the result can be an arbitrary Python
        # value.
306 307 308 309 310 311
        #
        # This must only be called when it is assured that all
        # sub-expressions have a valid constant_result value.  The
        # ConstantFolding transform will do this.
        pass

312 313 314 315
    def has_constant_result(self):
        return self.constant_result is not constant_value_not_set and \
               self.constant_result is not not_a_constant

316 317 318
    def compile_time_value(self, denv):
        #  Return value of compile-time expression, or report error.
        error(self.pos, "Invalid compile-time expression")
319

320 321 322
    def compile_time_value_error(self, e):
        error(self.pos, "Error in compile-time expression: %s: %s" % (
            e.__class__.__name__, e))
323

William Stein's avatar
William Stein committed
324
    # ------------- Declaration Analysis ----------------
325

William Stein's avatar
William Stein committed
326 327
    def analyse_target_declaration(self, env):
        error(self.pos, "Cannot assign to or delete this")
328

William Stein's avatar
William Stein committed
329
    # ------------- Expression Analysis ----------------
330

William Stein's avatar
William Stein committed
331 332 333 334 335 336
    def analyse_const_expression(self, env):
        #  Called during the analyse_declarations phase of a
        #  constant expression. Analyses the expression's type,
        #  checks whether it is a legal const expression,
        #  and determines its value.
        self.analyse_types(env)
337
        return self.check_const()
338

William Stein's avatar
William Stein committed
339 340
    def analyse_expressions(self, env):
        #  Convenience routine performing both the Type
341
        #  Analysis and Temp Allocation phases for a whole
William Stein's avatar
William Stein committed
342 343
        #  expression.
        self.analyse_types(env)
344

345
    def analyse_target_expression(self, env, rhs):
William Stein's avatar
William Stein committed
346 347 348 349
        #  Convenience routine performing both the Type
        #  Analysis and Temp Allocation phases for the LHS of
        #  an assignment.
        self.analyse_target_types(env)
350

William Stein's avatar
William Stein committed
351 352 353 354 355
    def analyse_boolean_expression(self, env):
        #  Analyse expression and coerce to a boolean.
        self.analyse_types(env)
        bool = self.coerce_to_boolean(env)
        return bool
356

William Stein's avatar
William Stein committed
357 358 359 360 361 362 363 364 365
    def analyse_temp_boolean_expression(self, env):
        #  Analyse boolean expression and coerce result into
        #  a temporary. This is used when a branch is to be
        #  performed on the result and we won't have an
        #  opportunity to ensure disposal code is executed
        #  afterwards. By forcing the result into a temporary,
        #  we ensure that all disposal has been done by the
        #  time we get the result.
        self.analyse_types(env)
Stefan Behnel's avatar
Stefan Behnel committed
366 367
        return self.coerce_to_boolean(env).coerce_to_simple(env)

368
    # --------------- Type Inference -----------------
369

Robert Bradshaw's avatar
Robert Bradshaw committed
370
    def type_dependencies(self, env):
371 372 373 374
        # Returns the list of entries whose types must be determined
        # before the type of self can be infered.
        if hasattr(self, 'type') and self.type is not None:
            return ()
Robert Bradshaw's avatar
Robert Bradshaw committed
375
        return sum([node.type_dependencies(env) for node in self.subexpr_nodes()], ())
376

377
    def infer_type(self, env):
378 379
        # Attempt to deduce the type of self.
        # Differs from analyse_types as it avoids unnecessary
380 381 382 383 384 385 386 387
        # analysis of subexpressions, but can assume everything
        # in self.type_dependencies() has been resolved.
        if hasattr(self, 'type') and self.type is not None:
            return self.type
        elif hasattr(self, 'entry') and self.entry is not None:
            return self.entry.type
        else:
            self.not_implemented("infer_type")
388

389 390 391
    def nonlocally_immutable(self):
        # Returns whether this variable is a safe reference, i.e.
        # can't be modified as part of globals or closures.
392
        return self.is_literal or self.is_temp or self.type.is_array or self.type.is_cfunction
393

William Stein's avatar
William Stein committed
394
    # --------------- Type Analysis ------------------
395

William Stein's avatar
William Stein committed
396 397 398 399
    def analyse_as_module(self, env):
        # If this node can be interpreted as a reference to a
        # cimported module, return its scope, else None.
        return None
400

401 402 403 404
    def analyse_as_type(self, env):
        # If this node can be interpreted as a reference to a
        # type, return that type, else None.
        return None
405

William Stein's avatar
William Stein committed
406 407 408 409
    def analyse_as_extension_type(self, env):
        # If this node can be interpreted as a reference to an
        # extension type, return its type, else None.
        return None
410

William Stein's avatar
William Stein committed
411 412
    def analyse_types(self, env):
        self.not_implemented("analyse_types")
413

William Stein's avatar
William Stein committed
414 415
    def analyse_target_types(self, env):
        self.analyse_types(env)
416

417
    def nogil_check(self, env):
418 419 420
        # By default, any expression based on Python objects is
        # prevented in nogil environments.  Subtypes must override
        # this if they can work without the GIL.
421
        if self.type and self.type.is_pyobject:
422
            self.gil_error()
423

424 425 426 427
    def gil_assignment_check(self, env):
        if env.nogil and self.type.is_pyobject:
            error(self.pos, "Assignment of Python object not allowed without gil")

William Stein's avatar
William Stein committed
428 429
    def check_const(self):
        self.not_const()
430
        return False
431

William Stein's avatar
William Stein committed
432 433
    def not_const(self):
        error(self.pos, "Not allowed in a constant expression")
434

William Stein's avatar
William Stein committed
435 436
    def check_const_addr(self):
        self.addr_not_const()
437
        return False
438

William Stein's avatar
William Stein committed
439 440
    def addr_not_const(self):
        error(self.pos, "Address is not constant")
441

William Stein's avatar
William Stein committed
442
    # ----------------- Result Allocation -----------------
443

William Stein's avatar
William Stein committed
444 445 446 447 448 449
    def result_in_temp(self):
        #  Return true if result is in a temporary owned by
        #  this node or one of its subexpressions. Overridden
        #  by certain nodes which can share the result of
        #  a subnode.
        return self.is_temp
450

William Stein's avatar
William Stein committed
451 452 453
    def target_code(self):
        #  Return code fragment for use as LHS of a C assignment.
        return self.calculate_result_code()
454

William Stein's avatar
William Stein committed
455 456
    def calculate_result_code(self):
        self.not_implemented("calculate_result_code")
457

Robert Bradshaw's avatar
Robert Bradshaw committed
458 459 460
#    def release_target_temp(self, env):
#        #  Release temporaries used by LHS of an assignment.
#        self.release_subexpr_temps(env)
William Stein's avatar
William Stein committed
461

462 463
    def allocate_temp_result(self, code):
        if self.temp_code:
464
            raise RuntimeError("Temp allocated multiple times in %r: %r" % (self.__class__.__name__, self.pos))
465 466 467 468 469
        type = self.type
        if not type.is_void:
            if type.is_pyobject:
                type = PyrexTypes.py_object_type
            self.temp_code = code.funcstate.allocate_temp(
470
                type, manage_ref=self.use_managed_ref)
471 472 473 474 475
        else:
            self.temp_code = None

    def release_temp_result(self, code):
        if not self.temp_code:
476 477 478
            if not self.result_is_used:
                # not used anyway, so ignore if not set up
                return
479 480 481 482 483 484 485 486 487 488
            if self.old_temp:
                raise RuntimeError("temp %s released multiple times in %s" % (
                        self.old_temp, self.__class__.__name__))
            else:
                raise RuntimeError("no temp, but release requested in %s" % (
                        self.__class__.__name__))
        code.funcstate.release_temp(self.temp_code)
        self.old_temp = self.temp_code
        self.temp_code = None

William Stein's avatar
William Stein committed
489
    # ---------------- Code Generation -----------------
490

William Stein's avatar
William Stein committed
491
    def make_owned_reference(self, code):
492 493 494 495
        """
        If result is a pyobject, make sure we own a reference to it.
        If the result is in a temp, it is already a new reference.
        """
William Stein's avatar
William Stein committed
496
        if self.type.is_pyobject and not self.result_in_temp():
497
            code.put_incref(self.result(), self.ctype())
498

499 500 501 502 503 504 505 506
    def make_owned_memoryviewslice(self, code):
        """
        Make sure we own the reference to this memoryview slice.
        """
        if not self.result_in_temp():
            code.put_incref_memoryviewslice(self.result(),
                                            have_gil=self.in_nogil_context)

William Stein's avatar
William Stein committed
507
    def generate_evaluation_code(self, code):
508
        code.mark_pos(self.pos)
509

William Stein's avatar
William Stein committed
510 511 512 513
        #  Generate code to evaluate this node and
        #  its sub-expressions, and dispose of any
        #  temporary results of its sub-expressions.
        self.generate_subexpr_evaluation_code(code)
514 515 516 517

        if self.is_temp:
            self.allocate_temp_result(code)

William Stein's avatar
William Stein committed
518 519
        self.generate_result_code(code)
        if self.is_temp:
520 521
            # If we are temp we do not need to wait until this node is disposed
            # before disposing children.
William Stein's avatar
William Stein committed
522
            self.generate_subexpr_disposal_code(code)
523
            self.free_subexpr_temps(code)
524

William Stein's avatar
William Stein committed
525 526 527
    def generate_subexpr_evaluation_code(self, code):
        for node in self.subexpr_nodes():
            node.generate_evaluation_code(code)
528

William Stein's avatar
William Stein committed
529 530
    def generate_result_code(self, code):
        self.not_implemented("generate_result_code")
531

532 533
    def generate_disposal_code(self, code):
        if self.is_temp:
534 535 536 537 538 539
            if self.result():
                if self.type.is_pyobject:
                    code.put_decref_clear(self.result(), self.ctype())
                elif self.type.is_memoryviewslice:
                    code.put_xdecref_memoryviewslice(
                            self.result(), have_gil=not self.in_nogil_context)
William Stein's avatar
William Stein committed
540
        else:
541
            # Already done if self.is_temp
542
            self.generate_subexpr_disposal_code(code)
543

William Stein's avatar
William Stein committed
544 545 546 547 548
    def generate_subexpr_disposal_code(self, code):
        #  Generate code to dispose of temporary results
        #  of all sub-expressions.
        for node in self.subexpr_nodes():
            node.generate_disposal_code(code)
549

William Stein's avatar
William Stein committed
550 551 552
    def generate_post_assignment_code(self, code):
        if self.is_temp:
            if self.type.is_pyobject:
553
                code.putln("%s = 0;" % self.result())
554 555 556
            elif self.type.is_memoryviewslice:
                code.putln("%s.memview = NULL;" % self.result())
                code.putln("%s.data = NULL;" % self.result())
William Stein's avatar
William Stein committed
557 558
        else:
            self.generate_subexpr_disposal_code(code)
559

William Stein's avatar
William Stein committed
560 561
    def generate_assignment_code(self, rhs, code):
        #  Stub method for nodes which are not legal as
562
        #  the LHS of an assignment. An error will have
William Stein's avatar
William Stein committed
563 564
        #  been reported earlier.
        pass
565

William Stein's avatar
William Stein committed
566 567 568 569 570
    def generate_deletion_code(self, code):
        #  Stub method for nodes that are not legal as
        #  the argument of a del statement. An error
        #  will have been reported earlier.
        pass
571 572

    def free_temps(self, code):
573 574 575 576
        if self.is_temp:
            if not self.type.is_void:
                self.release_temp_result(code)
        else:
577
            self.free_subexpr_temps(code)
578

579 580 581 582
    def free_subexpr_temps(self, code):
        for sub in self.subexpr_nodes():
            sub.free_temps(code)

583 584 585
    def generate_function_definitions(self, env, code):
        pass

586
    # ---------------- Annotation ---------------------
587

588 589 590
    def annotate(self, code):
        for node in self.subexpr_nodes():
            node.annotate(code)
591

William Stein's avatar
William Stein committed
592
    # ----------------- Coercion ----------------------
593

William Stein's avatar
William Stein committed
594 595 596 597 598 599 600 601
    def coerce_to(self, dst_type, env):
        #   Coerce the result so that it can be assigned to
        #   something of type dst_type. If processing is necessary,
        #   wraps this node in a coercion node and returns that.
        #   Otherwise, returns this node unchanged.
        #
        #   This method is called during the analyse_expressions
        #   phase of the src_node's processing.
602 603 604 605 606 607 608 609
        #
        #   Note that subclasses that override this (especially
        #   ConstNodes) must not (re-)set their own .type attribute
        #   here.  Since expression nodes may turn up in different
        #   places in the tree (e.g. inside of CloneNodes in cascaded
        #   assignments), this method must return a new node instance
        #   if it changes the type.
        #
William Stein's avatar
William Stein committed
610 611 612 613
        src = self
        src_type = self.type
        src_is_py_type = src_type.is_pyobject
        dst_is_py_type = dst_type.is_pyobject
614

615 616 617
        if self.check_for_coercion_error(dst_type):
            return self

618
        if dst_type.is_reference and not src_type.is_reference:
619
            dst_type = dst_type.ref_base_type
620

Robert Bradshaw's avatar
Robert Bradshaw committed
621 622 623
        if src_type.is_const:
            src_type = src_type.const_base_type

624
        if src_type.is_fused or dst_type.is_fused:
625 626 627 628 629 630 631
            # See if we are coercing a fused function to a pointer to a
            # specialized function
            if (src_type.is_cfunction and not dst_type.is_fused and
                    dst_type.is_ptr and dst_type.base_type.is_cfunction):

                dst_type = dst_type.base_type

632
                for signature in src_type.get_all_specialized_function_types():
633
                    if signature.same_as(dst_type):
Mark Florisson's avatar
Mark Florisson committed
634 635 636 637
                        src.type = signature
                        src.entry = src.type.entry
                        src.entry.used = True
                        return self
638

639
            if src_type.is_fused:
Mark Florisson's avatar
Mark Florisson committed
640
                error(self.pos, "Type is not specialized")
641 642 643
            else:
                error(self.pos, "Cannot coerce to a type that is not specialized")

644 645 646
            self.type = error_type
            return self

647 648 649 650 651
        if self.coercion_type is not None:
            # This is purely for error checking purposes!
            node = NameNode(self.pos, name='', type=self.coercion_type)
            node.coerce_to(dst_type, env)

652
        if dst_type.is_memoryviewslice:
653
            import MemoryView
654
            if not src.type.is_memoryviewslice:
655 656
                if src.type.is_pyobject:
                    src = CoerceToMemViewSliceNode(src, dst_type, env)
657 658 659
                elif src.type.is_array:
                    src = CythonArrayNode.from_carray(src, env).coerce_to(
                                                            dst_type, env)
660
                elif not src_type.is_error:
661 662 663
                    error(self.pos,
                          "Cannot convert '%s' to memoryviewslice" %
                                                                (src_type,))
664 665
            elif not MemoryView.src_conforms_to_dst(
                        src.type, dst_type, broadcast=self.memslice_broadcast):
666 667 668 669 670 671 672 673
                if src.type.dtype.same_as(dst_type.dtype):
                    msg = "Memoryview '%s' not conformable to memoryview '%s'."
                    tup = src.type, dst_type
                else:
                    msg = "Different base types for memoryviews (%s, %s)"
                    tup = src.type.dtype, dst_type.dtype

                error(self.pos, msg % tup)
674

675
        elif dst_type.is_pyobject:
William Stein's avatar
William Stein committed
676
            if not src.type.is_pyobject:
677 678 679 680
                if dst_type is bytes_type and src.type.is_int:
                    src = CoerceIntToBytesNode(src, env)
                else:
                    src = CoerceToPyTypeNode(src, env)
William Stein's avatar
William Stein committed
681
            if not src.type.subtype_of(dst_type):
682 683
                if not isinstance(src, NoneNode):
                    src = PyTypeTestNode(src, dst_type, env)
William Stein's avatar
William Stein committed
684 685
        elif src.type.is_pyobject:
            src = CoerceFromPyTypeNode(dst_type, src, env)
686
        elif (dst_type.is_complex
687 688
              and src_type != dst_type
              and dst_type.assignable_from(src_type)):
689
            src = CoerceToComplexNode(src, dst_type, env)
William Stein's avatar
William Stein committed
690
        else: # neither src nor dst are py types
691
            # Added the string comparison, since for c types that
692
            # is enough, but Cython gets confused when the types are
693
            # in different pxi files.
694
            if not (str(src.type) == str(dst_type) or dst_type.assignable_from(src_type)):
695
                self.fail_assignment(dst_type)
William Stein's avatar
William Stein committed
696 697
        return src

698 699 700 701 702 703 704 705 706 707 708 709 710 711 712
    def fail_assignment(self, dst_type):
        error(self.pos, "Cannot assign type '%s' to '%s'" % (self.type, dst_type))

    def check_for_coercion_error(self, dst_type, fail=False, default=None):
        if fail and not default:
            default = "Cannot assign type '%(FROM)s' to '%(TO)s'"
        message = find_coercion_error((self.type, dst_type), default)
        if message is not None:
            error(self.pos, message % {'FROM': self.type, 'TO': dst_type})
            return True
        if fail:
            self.fail_assignment(dst_type)
            return True
        return False

William Stein's avatar
William Stein committed
713 714 715 716 717 718
    def coerce_to_pyobject(self, env):
        return self.coerce_to(PyrexTypes.py_object_type, env)

    def coerce_to_boolean(self, env):
        #  Coerce result to something acceptable as
        #  a boolean value.
719 720 721 722 723 724 725

        # if it's constant, calculate the result now
        if self.has_constant_result():
            bool_value = bool(self.constant_result)
            return BoolNode(self.pos, value=bool_value,
                            constant_result=bool_value)

William Stein's avatar
William Stein committed
726 727 728 729
        type = self.type
        if type.is_pyobject or type.is_ptr or type.is_float:
            return CoerceToBooleanNode(self, env)
        else:
730
            if not (type.is_int or type.is_enum or type.is_error):
731
                error(self.pos,
William Stein's avatar
William Stein committed
732 733
                    "Type '%s' not acceptable as a boolean" % type)
            return self
734

William Stein's avatar
William Stein committed
735 736 737 738 739 740
    def coerce_to_integer(self, env):
        # If not already some C integer type, coerce to longint.
        if self.type.is_int:
            return self
        else:
            return self.coerce_to(PyrexTypes.c_long_type, env)
741

William Stein's avatar
William Stein committed
742 743 744 745 746 747
    def coerce_to_temp(self, env):
        #  Ensure that the result is in a temporary.
        if self.result_in_temp():
            return self
        else:
            return CoerceToTempNode(self, env)
748

William Stein's avatar
William Stein committed
749 750 751 752 753 754
    def coerce_to_simple(self, env):
        #  Ensure that the result is simple (see is_simple).
        if self.is_simple():
            return self
        else:
            return self.coerce_to_temp(env)
755

William Stein's avatar
William Stein committed
756 757 758 759 760 761
    def is_simple(self):
        #  A node is simple if its result is something that can
        #  be referred to without performing any operations, e.g.
        #  a constant, local var, C global var, struct member
        #  reference, or temporary.
        return self.result_in_temp()
762 763

    def may_be_none(self):
764 765
        if self.type and not (self.type.is_pyobject or
                              self.type.is_memoryviewslice):
766 767 768 769
            return False
        if self.constant_result not in (not_a_constant, constant_value_not_set):
            return self.constant_result is not None
        return True
770

771
    def as_cython_attribute(self):
772
        return None
William Stein's avatar
William Stein committed
773

774
    def as_none_safe_node(self, message, error="PyExc_TypeError", format_args=()):
775 776 777
        # Wraps the node in a NoneCheckNode if it is not known to be
        # not-None (e.g. because it is a Python literal).
        if self.may_be_none():
778
            return NoneCheckNode(self, error, message, format_args)
779 780 781 782
        else:
            return self


William Stein's avatar
William Stein committed
783
class AtomicExprNode(ExprNode):
784 785
    #  Abstract base class for expression nodes which have
    #  no sub-expressions.
786

787 788 789
    subexprs = []

    # Override to optimize -- we know we have no children
790 791 792 793
    def generate_subexpr_evaluation_code(self, code):
        pass
    def generate_subexpr_disposal_code(self, code):
        pass
794

795
class PyConstNode(AtomicExprNode):
William Stein's avatar
William Stein committed
796
    #  Abstract base class for constant Python values.
797

798
    is_literal = 1
799
    type = py_object_type
800

William Stein's avatar
William Stein committed
801 802
    def is_simple(self):
        return 1
803 804 805 806

    def may_be_none(self):
        return False

William Stein's avatar
William Stein committed
807
    def analyse_types(self, env):
808
        pass
809

William Stein's avatar
William Stein committed
810 811 812 813 814 815 816 817 818
    def calculate_result_code(self):
        return self.value

    def generate_result_code(self, code):
        pass


class NoneNode(PyConstNode):
    #  The constant value None
819

820
    is_none = 1
William Stein's avatar
William Stein committed
821
    value = "Py_None"
822 823

    constant_result = None
824

825
    nogil_check = None
826

827 828
    def compile_time_value(self, denv):
        return None
829 830 831 832 833

    def may_be_none(self):
        return True


William Stein's avatar
William Stein committed
834 835
class EllipsisNode(PyConstNode):
    #  '...' in a subscript list.
836

William Stein's avatar
William Stein committed
837 838
    value = "Py_Ellipsis"

839 840
    constant_result = Ellipsis

841 842 843
    def compile_time_value(self, denv):
        return Ellipsis

William Stein's avatar
William Stein committed
844

845
class ConstNode(AtomicExprNode):
William Stein's avatar
William Stein committed
846 847 848
    # Abstract base type for literal constant nodes.
    #
    # value     string      C code fragment
849

William Stein's avatar
William Stein committed
850
    is_literal = 1
851
    nogil_check = None
852

William Stein's avatar
William Stein committed
853 854
    def is_simple(self):
        return 1
855

856 857 858
    def nonlocally_immutable(self):
        return 1

859 860 861
    def may_be_none(self):
        return False

William Stein's avatar
William Stein committed
862 863
    def analyse_types(self, env):
        pass # Types are held in class variables
864

William Stein's avatar
William Stein committed
865
    def check_const(self):
866
        return True
867

868
    def get_constant_c_result_code(self):
869 870
        return self.calculate_result_code()

William Stein's avatar
William Stein committed
871 872 873 874 875 876 877
    def calculate_result_code(self):
        return str(self.value)

    def generate_result_code(self, code):
        pass


878 879 880
class BoolNode(ConstNode):
    type = PyrexTypes.c_bint_type
    #  The constant value True or False
881 882 883 884

    def calculate_constant_result(self):
        self.constant_result = self.value

885 886
    def compile_time_value(self, denv):
        return self.value
887

888
    def calculate_result_code(self):
889
        return str(int(self.value))
890

891

William Stein's avatar
William Stein committed
892 893
class NullNode(ConstNode):
    type = PyrexTypes.c_null_ptr_type
894
    value = "NULL"
895
    constant_result = 0
William Stein's avatar
William Stein committed
896

897
    def get_constant_c_result_code(self):
898 899
        return self.value

William Stein's avatar
William Stein committed
900 901 902

class CharNode(ConstNode):
    type = PyrexTypes.c_char_type
903 904 905

    def calculate_constant_result(self):
        self.constant_result = ord(self.value)
906

907
    def compile_time_value(self, denv):
908
        return ord(self.value)
909

William Stein's avatar
William Stein committed
910
    def calculate_result_code(self):
911
        return "'%s'" % StringEncoding.escape_char(self.value)
William Stein's avatar
William Stein committed
912 913 914


class IntNode(ConstNode):
915 916 917

    # unsigned     "" or "U"
    # longness     "" or "L" or "LL"
918
    # is_c_literal   True/False/None   creator considers this a C integer literal
919 920 921

    unsigned = ""
    longness = ""
922
    is_c_literal = None # unknown
923 924 925

    def __init__(self, pos, **kwds):
        ExprNode.__init__(self, pos, **kwds)
Robert Bradshaw's avatar
Robert Bradshaw committed
926
        if 'type' not in kwds:
927 928 929 930 931 932 933 934
            self.type = self.find_suitable_type_for_value()

    def find_suitable_type_for_value(self):
        if self.constant_result is constant_value_not_set:
            try:
                self.calculate_constant_result()
            except ValueError:
                pass
935 936 937 938
        # we ignore 'is_c_literal = True' and instead map signed 32bit
        # integers as C long values
        if self.is_c_literal or \
               self.constant_result in (constant_value_not_set, not_a_constant) or \
939 940 941 942 943 944 945 946 947 948 949 950 951 952 953 954
               self.unsigned or self.longness == 'LL':
            # clearly a C literal
            rank = (self.longness == 'LL') and 2 or 1
            suitable_type = PyrexTypes.modifiers_and_name_to_type[not self.unsigned, rank, "int"]
            if self.type:
                suitable_type = PyrexTypes.widest_numeric_type(suitable_type, self.type)
        else:
            # C literal or Python literal - split at 32bit boundary
            if self.constant_result >= -2**31 and self.constant_result < 2**31:
                if self.type and self.type.is_int:
                    suitable_type = self.type
                else:
                    suitable_type = PyrexTypes.c_long_type
            else:
                suitable_type = PyrexTypes.py_object_type
        return suitable_type
William Stein's avatar
William Stein committed
955

956
    def coerce_to(self, dst_type, env):
957
        if self.type is dst_type:
958
            return self
959
        elif dst_type.is_float:
960
            if self.constant_result is not not_a_constant:
961 962
                return FloatNode(self.pos, value='%d.0' % int(self.constant_result), type=dst_type,
                                 constant_result=float(self.constant_result))
963 964 965
            else:
                return FloatNode(self.pos, value=self.value, type=dst_type,
                                 constant_result=not_a_constant)
966
        if dst_type.is_numeric and not dst_type.is_complex:
967
            node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
968 969
                           type = dst_type, is_c_literal = True,
                           unsigned=self.unsigned, longness=self.longness)
970
            return node
971 972
        elif dst_type.is_pyobject:
            node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
973 974
                           type = PyrexTypes.py_object_type, is_c_literal = False,
                           unsigned=self.unsigned, longness=self.longness)
975
        else:
976 977
            # FIXME: not setting the type here to keep it working with
            # complex numbers. Should they be special cased?
978 979
            node = IntNode(self.pos, value=self.value, constant_result=self.constant_result,
                           unsigned=self.unsigned, longness=self.longness)
980 981 982
        # We still need to perform normal coerce_to processing on the
        # result, because we might be coercing to an extension type,
        # in which case a type test node will be needed.
983 984
        return ConstNode.coerce_to(node, dst_type, env)

985
    def coerce_to_boolean(self, env):
986 987 988 989
        return IntNode(
            self.pos, value=self.value,
            type = PyrexTypes.c_bint_type,
            unsigned=self.unsigned, longness=self.longness)
990

991
    def generate_evaluation_code(self, code):
992
        if self.type.is_pyobject:
993
            # pre-allocate a Python version of the number
994 995
            plain_integer_string = self.value_as_c_integer_string(plain_digits=True)
            self.result_code = code.get_py_num(plain_integer_string, self.longness)
996
        else:
997
            self.result_code = self.get_constant_c_result_code()
998

999
    def get_constant_c_result_code(self):
1000 1001 1002
        return self.value_as_c_integer_string() + self.unsigned + self.longness

    def value_as_c_integer_string(self, plain_digits=False):
1003 1004 1005 1006
        value = self.value
        if isinstance(value, basestring) and len(value) > 2:
            # must convert C-incompatible Py3 oct/bin notations
            if value[1] in 'oO':
1007 1008 1009 1010
                if plain_digits:
                    value = int(value[2:], 8)
                else:
                    value = value[0] + value[2:] # '0o123' => '0123'
1011 1012
            elif value[1] in 'bB':
                value = int(value[2:], 2)
1013 1014 1015
            elif plain_digits and value[1] in 'xX':
                value = int(value[2:], 16)
        return str(value)
1016 1017 1018

    def calculate_result_code(self):
        return self.result_code
William Stein's avatar
William Stein committed
1019

1020
    def calculate_constant_result(self):
1021
        self.constant_result = Utils.str_to_number(self.value)
1022

1023
    def compile_time_value(self, denv):
1024
        return Utils.str_to_number(self.value)
1025 1026


William Stein's avatar
William Stein committed
1027 1028 1029
class FloatNode(ConstNode):
    type = PyrexTypes.c_double_type

1030
    def calculate_constant_result(self):
1031
        self.constant_result = float(self.value)
1032

1033 1034
    def compile_time_value(self, denv):
        return float(self.value)
1035

Stefan Behnel's avatar
Stefan Behnel committed
1036
    def calculate_result_code(self):
1037 1038 1039 1040
        strval = self.value
        assert isinstance(strval, (str, unicode))
        cmpval = repr(float(strval))
        if cmpval == 'nan':
1041
            return "(Py_HUGE_VAL * 0)"
1042
        elif cmpval == 'inf':
1043
            return "Py_HUGE_VAL"
1044
        elif cmpval == '-inf':
1045
            return "(-Py_HUGE_VAL)"
Stefan Behnel's avatar
Stefan Behnel committed
1046 1047
        else:
            return strval
1048

William Stein's avatar
William Stein committed
1049

1050
class BytesNode(ConstNode):
1051 1052 1053 1054
    # A char* or bytes literal
    #
    # value      BytesLiteral

1055
    is_string_literal = True
1056 1057
    # start off as Python 'bytes' to support len() in O(1)
    type = bytes_type
1058 1059

    def compile_time_value(self, denv):
1060
        return self.value
1061

1062
    def analyse_as_type(self, env):
1063
        type = PyrexTypes.parse_basic_type(self.value)
1064
        if type is not None:
1065
            return type
1066 1067 1068 1069 1070 1071 1072
        from TreeFragment import TreeFragment
        pos = (self.pos[0], self.pos[1], self.pos[2]-7)
        declaration = TreeFragment(u"sizeof(%s)" % self.value, name=pos[0].filename, initial_pos=pos)
        sizeof_node = declaration.root.stats[0].expr
        sizeof_node.analyse_types(env)
        if isinstance(sizeof_node, SizeofTypeNode):
            return sizeof_node.arg_type
1073

1074 1075 1076
    def can_coerce_to_char_literal(self):
        return len(self.value) == 1

1077
    def coerce_to_boolean(self, env):
1078 1079
        # This is special because testing a C char* for truth directly
        # would yield the wrong result.
1080 1081
        bool_value = bool(self.value)
        return BoolNode(self.pos, value=bool_value, constant_result=bool_value)
1082

William Stein's avatar
William Stein committed
1083
    def coerce_to(self, dst_type, env):
1084 1085
        if self.type == dst_type:
            return self
1086
        if dst_type.is_int:
1087
            if not self.can_coerce_to_char_literal():
1088 1089
                error(self.pos, "Only single-character string literals can be coerced into ints.")
                return self
Stefan Behnel's avatar
Stefan Behnel committed
1090 1091
            if dst_type.is_unicode_char:
                error(self.pos, "Bytes literals cannot coerce to Py_UNICODE/Py_UCS4, use a unicode literal instead.")
1092
                return self
1093 1094
            return CharNode(self.pos, value=self.value)

1095
        node = BytesNode(self.pos, value=self.value)
1096 1097 1098 1099 1100 1101 1102 1103
        if dst_type.is_pyobject:
            if dst_type in (py_object_type, Builtin.bytes_type):
                node.type = Builtin.bytes_type
            else:
                self.check_for_coercion_error(dst_type, fail=True)
                return node
        elif dst_type == PyrexTypes.c_char_ptr_type:
            node.type = dst_type
1104 1105 1106 1107
            return node
        elif dst_type == PyrexTypes.c_uchar_ptr_type:
            node.type = PyrexTypes.c_char_ptr_type
            return CastNode(node, PyrexTypes.c_uchar_ptr_type)
1108 1109
        elif dst_type.assignable_from(PyrexTypes.c_char_ptr_type):
            node.type = dst_type
1110
            return node
1111

William Stein's avatar
William Stein committed
1112 1113 1114 1115 1116
        # We still need to perform normal coerce_to processing on the
        # result, because we might be coercing to an extension type,
        # in which case a type test node will be needed.
        return ConstNode.coerce_to(node, dst_type, env)

1117
    def generate_evaluation_code(self, code):
William Stein's avatar
William Stein committed
1118
        if self.type.is_pyobject:
1119
            self.result_code = code.get_py_string_const(self.value)
William Stein's avatar
William Stein committed
1120
        else:
1121
            self.result_code = code.get_string_const(self.value)
1122

1123
    def get_constant_c_result_code(self):
1124
        return None # FIXME
1125

1126 1127
    def calculate_result_code(self):
        return self.result_code
William Stein's avatar
William Stein committed
1128 1129


1130
class UnicodeNode(PyConstNode):
1131 1132
    # A Python unicode object
    #
1133 1134
    # value        EncodedString
    # bytes_value  BytesLiteral    the literal parsed as bytes string ('-3' unicode literals only)
Robert Bradshaw's avatar
Robert Bradshaw committed
1135

1136
    is_string_literal = True
1137
    bytes_value = None
1138
    type = unicode_type
1139

1140
    def coerce_to(self, dst_type, env):
1141 1142
        if dst_type is self.type:
            pass
Stefan Behnel's avatar
Stefan Behnel committed
1143
        elif dst_type.is_unicode_char:
1144
            if not self.can_coerce_to_char_literal():
Stefan Behnel's avatar
Stefan Behnel committed
1145
                error(self.pos, "Only single-character Unicode string literals or surrogate pairs can be coerced into Py_UCS4/Py_UNICODE.")
1146 1147
                return self
            int_value = ord(self.value)
Stefan Behnel's avatar
Stefan Behnel committed
1148
            return IntNode(self.pos, type=dst_type, value=str(int_value), constant_result=int_value)
1149
        elif not dst_type.is_pyobject:
1150 1151 1152
            if dst_type.is_string and self.bytes_value is not None:
                # special case: '-3' enforced unicode literal used in a C char* context
                return BytesNode(self.pos, value=self.bytes_value).coerce_to(dst_type, env)
Stefan Behnel's avatar
Stefan Behnel committed
1153
            error(self.pos, "Unicode literals do not support coercion to C types other than Py_UNICODE or Py_UCS4.")
1154 1155 1156 1157
        elif dst_type is not py_object_type:
            if not self.check_for_coercion_error(dst_type):
                self.fail_assignment(dst_type)
        return self
1158

1159 1160
    def can_coerce_to_char_literal(self):
        return len(self.value) == 1
Stefan Behnel's avatar
Stefan Behnel committed
1161 1162 1163
            ## or (len(self.value) == 2
            ##     and (0xD800 <= self.value[0] <= 0xDBFF)
            ##     and (0xDC00 <= self.value[1] <= 0xDFFF))
1164

1165 1166 1167 1168 1169 1170 1171 1172 1173 1174 1175 1176 1177 1178 1179 1180 1181
    def contains_surrogates(self):
        # Check if the unicode string contains surrogate code points
        # on a CPython platform with wide (UCS-4) or narrow (UTF-16)
        # Unicode, i.e. characters that would be spelled as two
        # separate code units on a narrow platform.
        for c in map(ord, self.value):
            if c > 65535: # can only happen on wide platforms
                return True
            # We only look for the first code unit (D800-DBFF) of a
            # surrogate pair - if we find one, the other one
            # (DC00-DFFF) is likely there, too.  If we don't find it,
            # any second code unit cannot make for a surrogate pair by
            # itself.
            if c >= 0xD800 and c <= 0xDBFF:
                return True
        return False

1182
    def generate_evaluation_code(self, code):
1183
        self.result_code = code.get_py_string_const(self.value)
1184 1185 1186

    def calculate_result_code(self):
        return self.result_code
1187

1188 1189
    def compile_time_value(self, env):
        return self.value
1190 1191


1192 1193 1194 1195
class StringNode(PyConstNode):
    # A Python str object, i.e. a byte string in Python 2.x and a
    # unicode string in Python 3.x
    #
1196 1197
    # value          BytesLiteral (or EncodedString with ASCII content)
    # unicode_value  EncodedString or None
1198
    # is_identifier  boolean
1199

1200
    type = str_type
1201
    is_string_literal = True
1202
    is_identifier = None
1203
    unicode_value = None
1204

1205
    def coerce_to(self, dst_type, env):
1206
        if dst_type is not py_object_type and not str_type.subtype_of(dst_type):
1207 1208 1209 1210 1211
#            if dst_type is Builtin.bytes_type:
#                # special case: bytes = 'str literal'
#                return BytesNode(self.pos, value=self.value)
            if not dst_type.is_pyobject:
                return BytesNode(self.pos, value=self.value).coerce_to(dst_type, env)
1212
            self.check_for_coercion_error(dst_type, fail=True)
1213
        return self
1214

1215 1216
    def can_coerce_to_char_literal(self):
        return not self.is_identifier and len(self.value) == 1
1217

1218
    def generate_evaluation_code(self, code):
1219
        self.result_code = code.get_py_string_const(
1220 1221
            self.value, identifier=self.is_identifier, is_str=True,
            unicode_value=self.unicode_value)
1222

1223
    def get_constant_c_result_code(self):
1224 1225
        return None

1226
    def calculate_result_code(self):
1227
        return self.result_code
1228

1229 1230
    def compile_time_value(self, env):
        return self.value
1231 1232


1233 1234 1235 1236
class IdentifierStringNode(StringNode):
    # A special str value that represents an identifier (bytes in Py2,
    # unicode in Py3).
    is_identifier = True
1237 1238


1239
class LongNode(AtomicExprNode):
William Stein's avatar
William Stein committed
1240 1241 1242
    #  Python long integer literal
    #
    #  value   string
1243

1244 1245
    type = py_object_type

1246
    def calculate_constant_result(self):
1247
        self.constant_result = Utils.str_to_number(self.value)
1248

1249
    def compile_time_value(self, denv):
1250
        return Utils.str_to_number(self.value)
1251

William Stein's avatar
William Stein committed
1252 1253
    def analyse_types(self, env):
        self.is_temp = 1
1254

1255 1256 1257
    def may_be_none(self):
        return False

1258 1259
    gil_message = "Constructing Python long int"

1260
    def generate_result_code(self, code):
William Stein's avatar
William Stein committed
1261
        code.putln(
1262
            '%s = PyLong_FromString((char *)"%s", 0, 0); %s' % (
1263
                self.result(),
William Stein's avatar
William Stein committed
1264
                self.value,
1265
                code.error_goto_if_null(self.result(), self.pos)))
1266
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
1267 1268


1269
class ImagNode(AtomicExprNode):
William Stein's avatar
William Stein committed
1270 1271 1272
    #  Imaginary number literal
    #
    #  value   float    imaginary part
1273

1274
    type = PyrexTypes.c_double_complex_type
1275 1276 1277

    def calculate_constant_result(self):
        self.constant_result = complex(0.0, self.value)
1278

1279 1280
    def compile_time_value(self, denv):
        return complex(0.0, self.value)
1281

William Stein's avatar
William Stein committed
1282
    def analyse_types(self, env):
1283 1284
        self.type.create_declaration_utility_code(env)

1285 1286 1287
    def may_be_none(self):
        return False

1288
    def coerce_to(self, dst_type, env):
1289 1290 1291
        if self.type is dst_type:
            return self
        node = ImagNode(self.pos, value=self.value)
1292
        if dst_type.is_pyobject:
1293 1294
            node.is_temp = 1
            node.type = PyrexTypes.py_object_type
1295 1296 1297
        # We still need to perform normal coerce_to processing on the
        # result, because we might be coercing to an extension type,
        # in which case a type test node will be needed.
1298
        return AtomicExprNode.coerce_to(node, dst_type, env)
1299 1300 1301

    gil_message = "Constructing complex number"

1302 1303 1304 1305 1306 1307
    def calculate_result_code(self):
        if self.type.is_pyobject:
            return self.result()
        else:
            return "%s(0, %r)" % (self.type.from_parts, float(self.value))

1308
    def generate_result_code(self, code):
1309 1310 1311 1312 1313 1314 1315
        if self.type.is_pyobject:
            code.putln(
                "%s = PyComplex_FromDoubles(0.0, %r); %s" % (
                    self.result(),
                    float(self.value),
                    code.error_goto_if_null(self.result(), self.pos)))
            code.put_gotref(self.py_result())
1316

William Stein's avatar
William Stein committed
1317

Danilo Freitas's avatar
Danilo Freitas committed
1318
class NewExprNode(AtomicExprNode):
1319 1320 1321

    # C++ new statement
    #
Robert Bradshaw's avatar
Robert Bradshaw committed
1322
    # cppclass              node                 c++ class to create
1323

Robert Bradshaw's avatar
Robert Bradshaw committed
1324
    type = None
1325

1326
    def infer_type(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
1327 1328
        type = self.cppclass.analyse_as_type(env)
        if type is None or not type.is_cpp_class:
Danilo Freitas's avatar
Danilo Freitas committed
1329
            error(self.pos, "new operator can only be applied to a C++ class")
Robert Bradshaw's avatar
Robert Bradshaw committed
1330
            self.type = error_type
Danilo Freitas's avatar
Danilo Freitas committed
1331
            return
Robert Bradshaw's avatar
Robert Bradshaw committed
1332
        self.cpp_check(env)
1333
        constructor = type.scope.lookup(u'<init>')
Danilo Freitas's avatar
Danilo Freitas committed
1334
        if constructor is None:
1335 1336
            func_type = PyrexTypes.CFuncType(type, [], exception_check='+')
            type.scope.declare_cfunction(u'<init>', func_type, self.pos)
1337
            constructor = type.scope.lookup(u'<init>')
1338
        self.class_type = type
DaniloFreitas's avatar
DaniloFreitas committed
1339
        self.entry = constructor
Robert Bradshaw's avatar
Robert Bradshaw committed
1340
        self.type = constructor.type
1341
        return self.type
1342

1343
    def analyse_types(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
1344 1345
        if self.type is None:
            self.infer_type(env)
1346 1347 1348 1349

    def may_be_none(self):
        return False

Danilo Freitas's avatar
Danilo Freitas committed
1350 1351
    def generate_result_code(self, code):
        pass
1352

Danilo Freitas's avatar
Danilo Freitas committed
1353
    def calculate_result_code(self):
1354
        return "new " + self.class_type.declaration_code("")
Danilo Freitas's avatar
Danilo Freitas committed
1355

William Stein's avatar
William Stein committed
1356

1357
class NameNode(AtomicExprNode):
William Stein's avatar
William Stein committed
1358 1359 1360 1361
    #  Reference to a local or global variable name.
    #
    #  name            string    Python name of the variable
    #  entry           Entry     Symbol table entry
1362
    #  type_entry      Entry     For extension type names, the original type entry
1363 1364
    #  cf_is_null      boolean   Is uninitialized before this node
    #  cf_maybe_null   boolean   Maybe uninitialized before this node
Vitja Makarov's avatar
Vitja Makarov committed
1365
    #  allow_null      boolean   Don't raise UnboundLocalError
1366
    #  nogil           boolean   Whether it is used in a nogil context
1367

1368 1369
    is_name = True
    is_cython_module = False
Robert Bradshaw's avatar
Robert Bradshaw committed
1370
    cython_attribute = None
1371
    lhs_of_first_assignment = False # TODO: remove me
1372
    is_used_as_rvalue = 0
1373
    entry = None
1374
    type_entry = None
1375 1376
    cf_maybe_null = True
    cf_is_null = False
Vitja Makarov's avatar
Vitja Makarov committed
1377
    allow_null = False
1378
    nogil = False
1379

1380
    def as_cython_attribute(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
1381
        return self.cython_attribute
1382

Robert Bradshaw's avatar
Robert Bradshaw committed
1383 1384
    def type_dependencies(self, env):
        if self.entry is None:
1385
            self.entry = env.lookup(self.name)
Robert Bradshaw's avatar
Robert Bradshaw committed
1386 1387 1388 1389
        if self.entry is not None and self.entry.type.is_unspecified:
            return (self.entry,)
        else:
            return ()
1390

Robert Bradshaw's avatar
Robert Bradshaw committed
1391 1392
    def infer_type(self, env):
        if self.entry is None:
1393
            self.entry = env.lookup(self.name)
Robert Bradshaw's avatar
Robert Bradshaw committed
1394 1395
        if self.entry is None:
            return py_object_type
Robert Bradshaw's avatar
Robert Bradshaw committed
1396 1397 1398
        elif (self.entry.type.is_extension_type or self.entry.type.is_builtin_type) and \
                self.name == self.entry.type.name:
            # Unfortunately the type attribute of type objects
1399
            # is used for the pointer to the type they represent.
Robert Bradshaw's avatar
Robert Bradshaw committed
1400
            return type_type
1401
        elif self.entry.type.is_cfunction:
1402 1403 1404 1405 1406 1407
            if self.entry.scope.is_builtin_scope:
                # special case: optimised builtin functions must be treated as Python objects
                return py_object_type
            else:
                # special case: referring to a C function must return its pointer
                return PyrexTypes.CPtrType(self.entry.type)
Robert Bradshaw's avatar
Robert Bradshaw committed
1408 1409
        else:
            return self.entry.type
1410

1411 1412 1413 1414
    def compile_time_value(self, denv):
        try:
            return denv.lookup(self.name)
        except KeyError:
Stefan Behnel's avatar
Stefan Behnel committed
1415
            error(self.pos, "Compile-time name '%s' not defined" % self.name)
1416 1417 1418 1419 1420

    def get_constant_c_result_code(self):
        if not self.entry or self.entry.type.is_pyobject:
            return None
        return self.entry.cname
1421

1422 1423 1424 1425 1426 1427 1428
    def coerce_to(self, dst_type, env):
        #  If coercing to a generic pyobject and this is a builtin
        #  C function with a Python equivalent, manufacture a NameNode
        #  referring to the Python builtin.
        #print "NameNode.coerce_to:", self.name, dst_type ###
        if dst_type is py_object_type:
            entry = self.entry
1429
            if entry and entry.is_cfunction:
1430 1431
                var_entry = entry.as_variable
                if var_entry:
1432
                    if var_entry.is_builtin and var_entry.is_const:
1433
                        var_entry = env.declare_builtin(var_entry.name, self.pos)
1434 1435 1436 1437
                    node = NameNode(self.pos, name = self.name)
                    node.entry = var_entry
                    node.analyse_rvalue_entry(env)
                    return node
1438

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1439
        return super(NameNode, self).coerce_to(dst_type, env)
1440

William Stein's avatar
William Stein committed
1441 1442 1443
    def analyse_as_module(self, env):
        # Try to interpret this as a reference to a cimported module.
        # Returns the module scope, or None.
1444 1445
        entry = self.entry
        if not entry:
1446
            entry = env.lookup(self.name)
William Stein's avatar
William Stein committed
1447 1448 1449
        if entry and entry.as_module:
            return entry.as_module
        return None
1450

1451
    def analyse_as_type(self, env):
1452 1453 1454 1455
        if self.cython_attribute:
            type = PyrexTypes.parse_basic_type(self.cython_attribute)
        else:
            type = PyrexTypes.parse_basic_type(self.name)
1456 1457
        if type:
            return type
1458 1459
        entry = self.entry
        if not entry:
1460
            entry = env.lookup(self.name)
1461 1462 1463 1464
        if entry and entry.is_type:
            return entry.type
        else:
            return None
1465

William Stein's avatar
William Stein committed
1466 1467 1468
    def analyse_as_extension_type(self, env):
        # Try to interpret this as a reference to an extension type.
        # Returns the extension type, or None.
1469 1470
        entry = self.entry
        if not entry:
1471
            entry = env.lookup(self.name)
William Stein's avatar
William Stein committed
1472
        if entry and entry.is_type and entry.type.is_extension_type:
1473 1474 1475
            return entry.type
        else:
            return None
1476

William Stein's avatar
William Stein committed
1477
    def analyse_target_declaration(self, env):
1478 1479
        if not self.entry:
            self.entry = env.lookup_here(self.name)
William Stein's avatar
William Stein committed
1480
        if not self.entry:
1481 1482
            if env.directives['warn.undeclared']:
                warning(self.pos, "implicit declaration of '%s'" % self.name, 1)
1483
            if env.directives['infer_types'] != False:
1484 1485 1486 1487
                type = unspecified_type
            else:
                type = py_object_type
            self.entry = env.declare_var(self.name, type, self.pos)
1488 1489
        if self.entry.is_declared_generic:
            self.result_ctype = py_object_type
1490

1491
    def analyse_types(self, env):
1492
        self.initialized_check = env.directives['initializedcheck']
1493
        if self.entry is None:
1494
            self.entry = env.lookup(self.name)
William Stein's avatar
William Stein committed
1495 1496
        if not self.entry:
            self.entry = env.declare_builtin(self.name, self.pos)
1497 1498 1499
        if not self.entry:
            self.type = PyrexTypes.error_type
            return
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1500 1501 1502 1503 1504 1505
        entry = self.entry
        if entry:
            entry.used = 1
            if entry.type.is_buffer:
                import Buffer
                Buffer.used_buffer_aux_vars(entry)
1506
        self.analyse_rvalue_entry(env)
1507

1508
    def analyse_target_types(self, env):
William Stein's avatar
William Stein committed
1509
        self.analyse_entry(env)
1510 1511 1512

        if (not self.is_lvalue() and self.entry.is_cfunction and
                self.entry.fused_cfunction and self.entry.as_variable):
1513
            # We need this for the fused 'def' TreeFragment
1514 1515 1516
            self.entry = self.entry.as_variable
            self.type = self.entry.type

Robert Bradshaw's avatar
Robert Bradshaw committed
1517 1518 1519 1520
        if self.type.is_const:
            error(self.pos, "Assignment to const '%s'" % self.name)
        if self.type.is_reference:
            error(self.pos, "Assignment to reference '%s'" % self.name)
1521 1522 1523 1524
        if not self.is_lvalue():
            error(self.pos, "Assignment to non-lvalue '%s'"
                % self.name)
            self.type = PyrexTypes.error_type
Stefan Behnel's avatar
Stefan Behnel committed
1525
        self.entry.used = 1
1526
        if self.entry.type.is_buffer:
1527 1528
            import Buffer
            Buffer.used_buffer_aux_vars(self.entry)
1529

1530 1531 1532 1533
    def analyse_rvalue_entry(self, env):
        #print "NameNode.analyse_rvalue_entry:", self.name ###
        #print "Entry:", self.entry.__dict__ ###
        self.analyse_entry(env)
1534
        entry = self.entry
1535

1536
        if entry.is_declared_generic:
William Stein's avatar
William Stein committed
1537
            self.result_ctype = py_object_type
1538

1539
        if entry.is_pyglobal or entry.is_builtin:
1540
            if entry.is_builtin and entry.is_const:
1541 1542 1543
                self.is_temp = 0
            else:
                self.is_temp = 1
1544
                env.use_utility_code(get_name_interned_utility_code)
1545

1546
            self.is_used_as_rvalue = 1
1547 1548 1549
        elif entry.type.is_memoryviewslice:
            self.is_temp = False
            self.is_used_as_rvalue = True
1550
            self.use_managed_ref = True
1551

1552
    def nogil_check(self, env):
1553
        self.nogil = True
1554 1555 1556
        if self.is_used_as_rvalue:
            entry = self.entry
            if entry.is_builtin:
1557
                if not entry.is_const: # cached builtins are ok
1558
                    self.gil_error()
1559
            elif entry.is_pyglobal:
1560
                self.gil_error()
1561 1562 1563 1564
            elif self.entry.type.is_memoryviewslice:
                if self.cf_is_null or self.cf_maybe_null:
                    import MemoryView
                    MemoryView.err_if_nogil_initialized_check(self.pos, env)
1565 1566 1567

    gil_message = "Accessing Python global or builtin"

1568 1569
    def analyse_entry(self, env):
        #print "NameNode.analyse_entry:", self.name ###
William Stein's avatar
William Stein committed
1570
        self.check_identifier_kind()
1571 1572 1573 1574
        entry = self.entry
        type = entry.type
        self.type = type

William Stein's avatar
William Stein committed
1575
    def check_identifier_kind(self):
1576 1577 1578
        # Check that this is an appropriate kind of name for use in an
        # expression.  Also finds the variable entry associated with
        # an extension type.
William Stein's avatar
William Stein committed
1579
        entry = self.entry
1580 1581
        if entry.is_type and entry.type.is_extension_type:
            self.type_entry = entry
1582
        if not (entry.is_const or entry.is_variable
Danilo Freitas's avatar
Danilo Freitas committed
1583 1584
            or entry.is_builtin or entry.is_cfunction
            or entry.is_cpp_class):
William Stein's avatar
William Stein committed
1585 1586 1587
                if self.entry.as_variable:
                    self.entry = self.entry.as_variable
                else:
1588
                    error(self.pos,
1589 1590
                          "'%s' is not a constant, variable or function identifier" % self.name)

William Stein's avatar
William Stein committed
1591 1592 1593
    def is_simple(self):
        #  If it's not a C variable, it'll be in a temp.
        return 1
1594

1595
    def may_be_none(self):
1596 1597
        if self.cf_state and self.type and (self.type.is_pyobject or
                                            self.type.is_memoryviewslice):
1598 1599 1600 1601 1602 1603 1604
            # gard against infinite recursion on self-dependencies
            if getattr(self, '_none_checking', False):
                # self-dependency - either this node receives a None
                # value from *another* node, or it can not reference
                # None at this point => safe to assume "not None"
                return False
            self._none_checking = True
1605 1606
            # evaluate control flow state to see if there were any
            # potential None values assigned to the node so far
1607
            may_be_none = False
1608 1609
            for assignment in self.cf_state:
                if assignment.rhs.may_be_none():
1610 1611 1612 1613
                    may_be_none = True
                    break
            del self._none_checking
            return may_be_none
1614 1615
        return super(NameNode, self).may_be_none()

1616
    def nonlocally_immutable(self):
1617 1618
        if ExprNode.nonlocally_immutable(self):
            return True
1619
        entry = self.entry
1620 1621 1622
        if not entry or entry.in_closure:
            return False
        return entry.is_local or entry.is_arg or entry.is_builtin or entry.is_readonly
1623

William Stein's avatar
William Stein committed
1624 1625
    def calculate_target_results(self, env):
        pass
1626

William Stein's avatar
William Stein committed
1627 1628
    def check_const(self):
        entry = self.entry
Robert Bradshaw's avatar
Robert Bradshaw committed
1629
        if entry is not None and not (entry.is_const or entry.is_cfunction or entry.is_builtin):
William Stein's avatar
William Stein committed
1630
            self.not_const()
1631 1632
            return False
        return True
1633

William Stein's avatar
William Stein committed
1634 1635
    def check_const_addr(self):
        entry = self.entry
1636
        if not (entry.is_cglobal or entry.is_cfunction or entry.is_builtin):
William Stein's avatar
William Stein committed
1637
            self.addr_not_const()
1638 1639
            return False
        return True
William Stein's avatar
William Stein committed
1640 1641 1642 1643 1644

    def is_lvalue(self):
        return self.entry.is_variable and \
            not self.entry.type.is_array and \
            not self.entry.is_readonly
1645

1646
    def is_addressable(self):
1647
        return self.entry.is_variable and not self.type.is_memoryviewslice
1648

William Stein's avatar
William Stein committed
1649 1650 1651 1652
    def is_ephemeral(self):
        #  Name nodes are never ephemeral, even if the
        #  result is in a temporary.
        return 0
1653

William Stein's avatar
William Stein committed
1654
    def calculate_result_code(self):
Stefan Behnel's avatar
Stefan Behnel committed
1655 1656
        entry = self.entry
        if not entry:
William Stein's avatar
William Stein committed
1657
            return "<error>" # There was an error earlier
Stefan Behnel's avatar
Stefan Behnel committed
1658
        return entry.cname
1659

William Stein's avatar
William Stein committed
1660
    def generate_result_code(self, code):
1661
        assert hasattr(self, 'entry')
William Stein's avatar
William Stein committed
1662 1663 1664
        entry = self.entry
        if entry is None:
            return # There was an error earlier
1665
        if entry.is_builtin and entry.is_const:
1666
            return # Lookup already cached
Stefan Behnel's avatar
Stefan Behnel committed
1667
        elif entry.is_pyclass_attr:
Vitja Makarov's avatar
Vitja Makarov committed
1668 1669 1670 1671 1672 1673
            assert entry.type.is_pyobject, "Python global or builtin not a Python object"
            interned_cname = code.intern_identifier(self.entry.name)
            if entry.is_builtin:
                namespace = Naming.builtins_cname
            else: # entry.is_pyglobal
                namespace = entry.scope.namespace_cname
1674 1675 1676 1677 1678 1679
            if not self.cf_is_null:
                code.putln(
                    '%s = PyObject_GetItem(%s, %s);' % (
                        self.result(),
                        namespace,
                        interned_cname))
1680 1681 1682 1683
                code.putln('if (unlikely(!%s)) {' % self.result())
                code.putln('PyErr_Clear();')
            code.putln(
                '%s = __Pyx_GetName(%s, %s);' % (
1684 1685 1686
                    self.result(),
                    Naming.module_cname,
                    interned_cname))
1687 1688
            if not self.cf_is_null:
                code.putln("}");
1689
            code.putln(code.error_goto_if_null(self.result(), self.pos))
Vitja Makarov's avatar
Vitja Makarov committed
1690
            code.put_gotref(self.py_result())
1691

1692
        elif entry.is_pyglobal or entry.is_builtin:
1693 1694
            assert entry.type.is_pyobject, "Python global or builtin not a Python object"
            interned_cname = code.intern_identifier(self.entry.name)
William Stein's avatar
William Stein committed
1695 1696 1697
            if entry.is_builtin:
                namespace = Naming.builtins_cname
            else: # entry.is_pyglobal
1698
                namespace = entry.scope.namespace_cname
1699
            code.globalstate.use_utility_code(get_name_interned_utility_code)
1700 1701
            code.putln(
                '%s = __Pyx_GetName(%s, %s); %s' % (
1702
                self.result(),
1703
                namespace,
1704
                interned_cname,
1705
                code.error_goto_if_null(self.result(), self.pos)))
1706
            code.put_gotref(self.py_result())
1707

1708
        elif entry.is_local or entry.in_closure or entry.from_closure or entry.type.is_memoryviewslice:
1709 1710 1711 1712
            # Raise UnboundLocalError for objects and memoryviewslices
            raise_unbound = (
                (self.cf_maybe_null or self.cf_is_null) and not self.allow_null)
            null_code = entry.type.check_for_null_code(entry.cname)
1713

1714 1715 1716
            memslice_check = entry.type.is_memoryviewslice and self.initialized_check

            if null_code and raise_unbound and (entry.type.is_pyobject or memslice_check):
1717
                code.put_error_if_unbound(self.pos, entry, self.in_nogil_context)
William Stein's avatar
William Stein committed
1718 1719

    def generate_assignment_code(self, rhs, code):
1720
        #print "NameNode.generate_assignment_code:", self.name ###
William Stein's avatar
William Stein committed
1721 1722 1723
        entry = self.entry
        if entry is None:
            return # There was an error earlier
1724 1725 1726 1727

        if (self.entry.type.is_ptr and isinstance(rhs, ListNode)
            and not self.lhs_of_first_assignment):
            error(self.pos, "Literal list must be assigned to pointer at time of declaration")
1728

1729 1730
        # is_pyglobal seems to be True for module level-globals only.
        # We use this to access class->tp_dict if necessary.
William Stein's avatar
William Stein committed
1731
        if entry.is_pyglobal:
1732 1733
            assert entry.type.is_pyobject, "Python global or builtin not a Python object"
            interned_cname = code.intern_identifier(self.entry.name)
1734
            namespace = self.entry.scope.namespace_cname
1735
            if entry.is_member:
Stefan Behnel's avatar
Stefan Behnel committed
1736
                # if the entry is a member we have to cheat: SetAttr does not work
1737
                # on types, so we create a descriptor which is then added to tp_dict
1738 1739 1740
                code.put_error_if_neg(self.pos,
                    'PyDict_SetItem(%s->tp_dict, %s, %s)' % (
                        namespace,
1741
                        interned_cname,
1742
                        rhs.py_result()))
1743 1744
                rhs.generate_disposal_code(code)
                rhs.free_temps(code)
1745
                # in Py2.6+, we need to invalidate the method cache
1746
                code.putln("PyType_Modified(%s);" %
Vitja Makarov's avatar
Vitja Makarov committed
1747
                            entry.scope.parent_type.typeptr_cname)
Stefan Behnel's avatar
Stefan Behnel committed
1748
            elif entry.is_pyclass_attr:
Vitja Makarov's avatar
Vitja Makarov committed
1749
                code.put_error_if_neg(self.pos,
Stefan Behnel's avatar
Stefan Behnel committed
1750
                    'PyObject_SetItem(%s, %s, %s)' % (
Vitja Makarov's avatar
Vitja Makarov committed
1751 1752 1753 1754 1755 1756
                        namespace,
                        interned_cname,
                        rhs.py_result()))
                rhs.generate_disposal_code(code)
                rhs.free_temps(code)
            else:
1757 1758 1759
                code.put_error_if_neg(self.pos,
                    'PyObject_SetAttr(%s, %s, %s)' % (
                        namespace,
1760
                        interned_cname,
1761
                        rhs.py_result()))
1762
                if debug_disposal_code:
Stefan Behnel's avatar
Stefan Behnel committed
1763 1764
                    print("NameNode.generate_assignment_code:")
                    print("...generating disposal code for %s" % rhs)
1765
                rhs.generate_disposal_code(code)
1766
                rhs.free_temps(code)
William Stein's avatar
William Stein committed
1767
        else:
1768
            if self.type.is_memoryviewslice:
1769
                self.generate_acquire_memoryviewslice(rhs, code)
1770

1771
            elif self.type.is_buffer:
1772 1773 1774 1775 1776 1777 1778 1779 1780
                # Generate code for doing the buffer release/acquisition.
                # This might raise an exception in which case the assignment (done
                # below) will not happen.
                #
                # The reason this is not in a typetest-like node is because the
                # variables that the acquired buffer info is stored to is allocated
                # per entry and coupled with it.
                self.generate_acquire_buffer(rhs, code)

1781
            if self.type.is_pyobject:
William Stein's avatar
William Stein committed
1782 1783 1784 1785
                #print "NameNode.generate_assignment_code: to", self.name ###
                #print "...from", rhs ###
                #print "...LHS type", self.type, "ctype", self.ctype() ###
                #print "...RHS type", rhs.type, "ctype", rhs.ctype() ###
1786 1787
                if self.use_managed_ref:
                    rhs.make_owned_reference(code)
1788
                    is_external_ref = entry.is_cglobal or self.entry.in_closure or self.entry.from_closure
1789 1790 1791 1792 1793 1794
                    if is_external_ref:
                        if not self.cf_is_null:
                            if self.cf_maybe_null:
                                code.put_xgotref(self.py_result())
                            else:
                                code.put_gotref(self.py_result())
1795 1796 1797
                    if entry.is_cglobal:
                        code.put_decref(self.result(), self.ctype())
                    else:
1798 1799
                        if not self.cf_is_null:
                            if self.cf_maybe_null:
1800
                                code.put_xdecref(self.result(), self.ctype())
1801 1802
                            else:
                                code.put_decref(self.result(), self.ctype())
1803
                    if is_external_ref:
1804
                        code.put_giveref(rhs.py_result())
1805
            if not self.type.is_memoryviewslice:
1806
                code.putln('%s = %s;' % (self.result(), rhs.result_as(self.ctype())))
1807 1808 1809 1810
                if debug_disposal_code:
                    print("NameNode.generate_assignment_code:")
                    print("...generating post-assignment code for %s" % rhs)
                rhs.generate_post_assignment_code(code)
1811 1812
            elif rhs.result_in_temp():
                rhs.generate_post_assignment_code(code)
1813

1814
            rhs.free_temps(code)
1815

1816 1817
    def generate_acquire_memoryviewslice(self, rhs, code):
        """
1818 1819
        Slices, coercions from objects, return values etc are new references.
        We have a borrowed reference in case of dst = src
1820 1821 1822 1823 1824 1825 1826 1827 1828
        """
        import MemoryView

        MemoryView.put_acquire_memoryviewslice(
            lhs_cname=self.result(),
            lhs_type=self.type,
            lhs_pos=self.pos,
            rhs=rhs,
            code=code,
1829 1830
            have_gil=not self.in_nogil_context,
            first_assignment=self.cf_is_null)
1831

1832
    def generate_acquire_buffer(self, rhs, code):
1833 1834 1835
        # rhstmp is only used in case the rhs is a complicated expression leading to
        # the object, to avoid repeating the same C expression for every reference
        # to the rhs. It does NOT hold a reference.
1836 1837 1838 1839 1840 1841 1842
        pretty_rhs = isinstance(rhs, NameNode) or rhs.is_temp
        if pretty_rhs:
            rhstmp = rhs.result_as(self.ctype())
        else:
            rhstmp = code.funcstate.allocate_temp(self.entry.type, manage_ref=False)
            code.putln('%s = %s;' % (rhstmp, rhs.result_as(self.ctype())))

1843
        import Buffer
1844
        Buffer.put_assign_to_buffer(self.result(), rhstmp, self.entry,
1845
                                    is_initialized=not self.lhs_of_first_assignment,
1846
                                    pos=self.pos, code=code)
1847

1848 1849 1850
        if not pretty_rhs:
            code.putln("%s = 0;" % rhstmp)
            code.funcstate.release_temp(rhstmp)
1851

William Stein's avatar
William Stein committed
1852 1853 1854
    def generate_deletion_code(self, code):
        if self.entry is None:
            return # There was an error earlier
1855
        elif self.entry.is_pyclass_attr:
Vitja Makarov's avatar
Vitja Makarov committed
1856
            namespace = self.entry.scope.namespace_cname
1857
            interned_cname = code.intern_identifier(self.entry.name)
Vitja Makarov's avatar
Vitja Makarov committed
1858
            code.put_error_if_neg(self.pos,
1859
                'PyObject_DelItem(%s, %s)' % (
Vitja Makarov's avatar
Vitja Makarov committed
1860
                    namespace,
1861
                    interned_cname))
1862 1863 1864 1865 1866
        elif self.entry.is_pyglobal:
            code.put_error_if_neg(self.pos,
                '__Pyx_DelAttrString(%s, "%s")' % (
                    Naming.module_cname,
                    self.entry.name))
1867
        elif self.entry.type.is_pyobject or self.entry.type.is_memoryviewslice:
1868 1869
            if not self.cf_is_null:
                if self.cf_maybe_null:
1870
                    code.put_error_if_unbound(self.pos, self.entry)
1871 1872 1873 1874 1875 1876 1877

                if self.entry.type.is_pyobject:
                    code.put_decref(self.result(), self.ctype())
                    code.putln('%s = NULL;' % self.result())
                else:
                    code.put_xdecref_memoryviewslice(self.entry.cname,
                                                     have_gil=not self.nogil)
Vitja Makarov's avatar
Vitja Makarov committed
1878
        else:
1879
            error(self.pos, "Deletion of C names not supported")
1880

1881 1882 1883 1884 1885 1886 1887
    def annotate(self, code):
        if hasattr(self, 'is_called') and self.is_called:
            pos = (self.pos[0], self.pos[1], self.pos[2] - len(self.name) - 1)
            if self.type.is_pyobject:
                code.annotate(pos, AnnotationItem('py_call', 'python function', size=len(self.name)))
            else:
                code.annotate(pos, AnnotationItem('c_call', 'c function', size=len(self.name)))
1888

1889
class BackquoteNode(ExprNode):
William Stein's avatar
William Stein committed
1890 1891 1892
    #  `expr`
    #
    #  arg    ExprNode
1893

1894
    type = py_object_type
1895

William Stein's avatar
William Stein committed
1896
    subexprs = ['arg']
1897

William Stein's avatar
William Stein committed
1898 1899 1900 1901
    def analyse_types(self, env):
        self.arg.analyse_types(env)
        self.arg = self.arg.coerce_to_pyobject(env)
        self.is_temp = 1
1902 1903 1904

    gil_message = "Backquote expression"

1905 1906 1907
    def calculate_constant_result(self):
        self.constant_result = repr(self.arg.constant_result)

William Stein's avatar
William Stein committed
1908 1909
    def generate_result_code(self, code):
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
1910
            "%s = PyObject_Repr(%s); %s" % (
1911
                self.result(),
William Stein's avatar
William Stein committed
1912
                self.arg.py_result(),
1913
                code.error_goto_if_null(self.result(), self.pos)))
1914
        code.put_gotref(self.py_result())
1915

William Stein's avatar
William Stein committed
1916

1917
class ImportNode(ExprNode):
William Stein's avatar
William Stein committed
1918
    #  Used as part of import statement implementation.
1919
    #  Implements result =
Haoyu Bai's avatar
Haoyu Bai committed
1920
    #    __import__(module_name, globals(), None, name_list, level)
William Stein's avatar
William Stein committed
1921
    #
Haoyu Bai's avatar
Haoyu Bai committed
1922 1923 1924
    #  module_name   StringNode            dotted name of module. Empty module
    #                       name means importing the parent package accourding
    #                       to level
1925
    #  name_list     ListNode or None      list of names to be imported
Haoyu Bai's avatar
Haoyu Bai committed
1926 1927 1928 1929 1930
    #  level         int                   relative import level:
    #                       -1: attempt both relative import and absolute import;
    #                        0: absolute import;
    #                       >0: the number of parent directories to search
    #                           relative to the current module.
1931 1932
    #                     None: decide the level according to language level and
    #                           directives
1933

1934
    type = py_object_type
1935

William Stein's avatar
William Stein committed
1936
    subexprs = ['module_name', 'name_list']
1937

William Stein's avatar
William Stein committed
1938
    def analyse_types(self, env):
1939
        if self.level is None:
1940 1941
            if (env.directives['py2_import'] or
                Future.absolute_import not in env.global_scope().context.future_directives):
1942 1943 1944
                self.level = -1
            else:
                self.level = 0
William Stein's avatar
William Stein committed
1945 1946 1947 1948
        self.module_name.analyse_types(env)
        self.module_name = self.module_name.coerce_to_pyobject(env)
        if self.name_list:
            self.name_list.analyse_types(env)
1949
            self.name_list.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
1950
        self.is_temp = 1
1951
        env.use_utility_code(UtilityCode.load_cached("Import", "ImportExport.c"))
1952 1953 1954

    gil_message = "Python import"

William Stein's avatar
William Stein committed
1955 1956 1957 1958 1959 1960
    def generate_result_code(self, code):
        if self.name_list:
            name_list_code = self.name_list.py_result()
        else:
            name_list_code = "0"
        code.putln(
Haoyu Bai's avatar
Haoyu Bai committed
1961
            "%s = __Pyx_Import(%s, %s, %d); %s" % (
1962
                self.result(),
William Stein's avatar
William Stein committed
1963 1964
                self.module_name.py_result(),
                name_list_code,
Haoyu Bai's avatar
Haoyu Bai committed
1965
                self.level,
1966
                code.error_goto_if_null(self.result(), self.pos)))
1967
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
1968 1969


1970
class IteratorNode(ExprNode):
William Stein's avatar
William Stein committed
1971
    #  Used as part of for statement implementation.
1972
    #
William Stein's avatar
William Stein committed
1973 1974 1975
    #  Implements result = iter(sequence)
    #
    #  sequence   ExprNode
1976

1977
    type = py_object_type
1978
    iter_func_ptr = None
1979
    counter_cname = None
1980
    cpp_iterator_cname = None
1981
    reversed = False      # currently only used for list/tuple types (see Optimize.py)
1982

William Stein's avatar
William Stein committed
1983
    subexprs = ['sequence']
1984

William Stein's avatar
William Stein committed
1985 1986
    def analyse_types(self, env):
        self.sequence.analyse_types(env)
1987 1988
        if (self.sequence.type.is_array or self.sequence.type.is_ptr) and \
                not self.sequence.type.is_string:
1989
            # C array iteration will be transformed later on
1990
            self.type = self.sequence.type
1991 1992
        elif self.sequence.type.is_cpp_class:
            self.analyse_cpp_types(env)
1993 1994
        else:
            self.sequence = self.sequence.coerce_to_pyobject(env)
1995 1996 1997
            if self.sequence.type is list_type or \
                   self.sequence.type is tuple_type:
                self.sequence = self.sequence.as_none_safe_node("'NoneType' object is not iterable")
William Stein's avatar
William Stein committed
1998
        self.is_temp = 1
1999 2000 2001

    gil_message = "Iterating over Python object"

2002 2003 2004 2005
    _func_iternext_type = PyrexTypes.CPtrType(PyrexTypes.CFuncType(
        PyrexTypes.py_object_type, [
            PyrexTypes.CFuncTypeArg("it", PyrexTypes.py_object_type, None),
            ]))
2006

2007 2008 2009
    def type_dependencies(self, env):
        return self.sequence.type_dependencies(env)

2010 2011
    def infer_type(self, env):
        sequence_type = self.sequence.infer_type(env)
2012
        if sequence_type.is_array or sequence_type.is_ptr:
2013 2014 2015 2016
            return sequence_type
        elif sequence_type.is_cpp_class:
            begin = sequence_type.scope.lookup("begin")
            if begin is not None:
2017
                return begin.type.return_type
2018 2019
        elif sequence_type.is_pyobject:
            return sequence_type
2020
        return py_object_type
2021
    
2022
    def analyse_cpp_types(self, env):
2023 2024 2025 2026 2027 2028
        sequence_type = self.sequence.type
        if sequence_type.is_ptr:
            sequence_type = sequence_type.base_type
        begin = sequence_type.scope.lookup("begin")
        end = sequence_type.scope.lookup("end")
        if (begin is None
2029 2030
            or not begin.type.is_cfunction
            or begin.type.args):
2031 2032 2033
            error(self.pos, "missing begin() on %s" % self.sequence.type)
            self.type = error_type
            return
2034
        if (end is None
2035 2036
            or not end.type.is_cfunction
            or end.type.args):
2037 2038 2039
            error(self.pos, "missing end() on %s" % self.sequence.type)
            self.type = error_type
            return
2040
        iter_type = begin.type.return_type
2041
        if iter_type.is_cpp_class:
2042 2043 2044
            if env.lookup_operator_for_types(
                    self.pos,
                    "!=",
2045
                    [iter_type, end.type.return_type]) is None:
2046 2047 2048
                error(self.pos, "missing operator!= on result of begin() on %s" % self.sequence.type)
                self.type = error_type
                return
2049
            if env.lookup_operator_for_types(self.pos, '++', [iter_type]) is None:
2050 2051 2052
                error(self.pos, "missing operator++ on result of begin() on %s" % self.sequence.type)
                self.type = error_type
                return
2053
            if env.lookup_operator_for_types(self.pos, '*', [iter_type]) is None:
2054 2055 2056 2057 2058
                error(self.pos, "missing operator* on result of begin() on %s" % self.sequence.type)
                self.type = error_type
                return
            self.type = iter_type
        elif iter_type.is_ptr:
2059
            if not (iter_type == end.type.return_type):
2060
                error(self.pos, "incompatible types for begin() and end()")
2061 2062 2063 2064 2065
            self.type = iter_type
        else:
            error(self.pos, "result type of begin() on %s must be a C++ class or pointer" % self.sequence.type)
            self.type = error_type
            return
2066

William Stein's avatar
William Stein committed
2067
    def generate_result_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
2068
        sequence_type = self.sequence.type
2069
        if sequence_type.is_cpp_class:
2070 2071 2072 2073 2074 2075 2076 2077
            if self.sequence.is_name:
                # safe: C++ won't allow you to reassign to class references
                begin_func = "%s.begin" % self.sequence.result()
            else:
                sequence_type = PyrexTypes.c_ptr_type(sequence_type)
                self.cpp_iterator_cname = code.funcstate.allocate_temp(sequence_type, manage_ref=False)
                code.putln("%s = &%s;" % (self.cpp_iterator_cname, self.sequence.result()))
                begin_func = "%s->begin" % self.cpp_iterator_cname
2078
            # TODO: Limit scope.
2079
            code.putln("%s = %s();" % (self.result(), begin_func))
2080
            return
Stefan Behnel's avatar
Stefan Behnel committed
2081
        if sequence_type.is_array or sequence_type.is_ptr:
2082
            raise InternalError("for in carray slice not transformed")
Stefan Behnel's avatar
Stefan Behnel committed
2083 2084
        is_builtin_sequence = sequence_type is list_type or \
                              sequence_type is tuple_type
2085 2086 2087
        if not is_builtin_sequence:
            # reversed() not currently optimised (see Optimize.py)
            assert not self.reversed, "internal error: reversed() only implemented for list/tuple objects"
Stefan Behnel's avatar
Stefan Behnel committed
2088 2089
        self.may_be_a_sequence = not sequence_type.is_builtin_type
        if self.may_be_a_sequence:
2090 2091 2092 2093
            code.putln(
                "if (PyList_CheckExact(%s) || PyTuple_CheckExact(%s)) {" % (
                    self.sequence.py_result(),
                    self.sequence.py_result()))
Stefan Behnel's avatar
Stefan Behnel committed
2094
        if is_builtin_sequence or self.may_be_a_sequence:
2095 2096
            self.counter_cname = code.funcstate.allocate_temp(
                PyrexTypes.c_py_ssize_t_type, manage_ref=False)
2097 2098 2099 2100 2101 2102 2103
            if self.reversed:
                if sequence_type is list_type:
                    init_value = 'PyList_GET_SIZE(%s) - 1' % self.result()
                else:
                    init_value = 'PyTuple_GET_SIZE(%s) - 1' % self.result()
            else:
                init_value = '0'
2104
            code.putln(
2105
                "%s = %s; __Pyx_INCREF(%s); %s = %s;" % (
2106 2107
                    self.result(),
                    self.sequence.py_result(),
2108 2109 2110 2111
                    self.result(),
                    self.counter_cname,
                    init_value
                    ))
2112
        if not is_builtin_sequence:
Stefan Behnel's avatar
Stefan Behnel committed
2113
            self.iter_func_ptr = code.funcstate.allocate_temp(self._func_iternext_type, manage_ref=False)
Stefan Behnel's avatar
Stefan Behnel committed
2114
            if self.may_be_a_sequence:
Stefan Behnel's avatar
Stefan Behnel committed
2115
                code.putln("%s = NULL;" % self.iter_func_ptr)
2116
                code.putln("} else {")
2117 2118
                code.put("%s = -1; " % self.counter_cname)
            code.putln("%s = PyObject_GetIter(%s); %s" % (
2119 2120 2121
                    self.result(),
                    self.sequence.py_result(),
                    code.error_goto_if_null(self.result(), self.pos)))
2122
            code.put_gotref(self.py_result())
2123
            code.putln("%s = Py_TYPE(%s)->tp_iternext;" % (self.iter_func_ptr, self.py_result()))
Stefan Behnel's avatar
Stefan Behnel committed
2124 2125 2126 2127
        if self.may_be_a_sequence:
            code.putln("}")

    def generate_next_sequence_item(self, test_name, result_name, code):
2128
        assert self.counter_cname, "internal error: counter_cname temp not prepared"
Stefan Behnel's avatar
Stefan Behnel committed
2129 2130 2131 2132 2133
        code.putln(
            "if (%s >= Py%s_GET_SIZE(%s)) break;" % (
                self.counter_cname,
                test_name,
                self.py_result()))
2134 2135 2136 2137
        if self.reversed:
            inc_dec = '--'
        else:
            inc_dec = '++'
2138
        code.putln("#if CYTHON_COMPILING_IN_CPYTHON")
Stefan Behnel's avatar
Stefan Behnel committed
2139
        code.putln(
2140
            "%s = Py%s_GET_ITEM(%s, %s); __Pyx_INCREF(%s); %s%s; %s" % (
Stefan Behnel's avatar
Stefan Behnel committed
2141 2142 2143 2144 2145
                result_name,
                test_name,
                self.py_result(),
                self.counter_cname,
                result_name,
2146
                self.counter_cname,
2147 2148 2149 2150
                inc_dec,
                # use the error label to avoid C compiler warnings if we only use it below
                code.error_goto_if_neg('0', self.pos)
                ))
2151 2152
        code.putln("#else")
        code.putln(
2153
            "%s = PySequence_ITEM(%s, %s); %s%s; %s" % (
2154 2155 2156 2157 2158 2159 2160
                result_name,
                self.py_result(),
                self.counter_cname,
                self.counter_cname,
                inc_dec,
                code.error_goto_if_null(result_name, self.pos)))
        code.putln("#endif")
Stefan Behnel's avatar
Stefan Behnel committed
2161 2162 2163

    def generate_iter_next_result_code(self, result_name, code):
        sequence_type = self.sequence.type
2164 2165
        if self.reversed:
            code.putln("if (%s < 0) break;" % self.counter_cname)
2166
        if sequence_type.is_cpp_class:
2167 2168 2169 2170
            if self.cpp_iterator_cname:
                end_func = "%s->end" % self.cpp_iterator_cname
            else:
                end_func = "%s.end" % self.sequence.result()
2171
            # TODO: Cache end() call?
2172
            code.putln("if (!(%s != %s())) break;" % (
2173
                            self.result(),
2174
                            end_func))
2175 2176 2177 2178 2179 2180
            code.putln("%s = *%s;" % (
                            result_name,
                            self.result()))
            code.putln("++%s;" % self.result())
            return
        elif sequence_type is list_type:
Stefan Behnel's avatar
Stefan Behnel committed
2181 2182 2183 2184 2185 2186 2187 2188
            self.generate_next_sequence_item('List', result_name, code)
            return
        elif sequence_type is tuple_type:
            self.generate_next_sequence_item('Tuple', result_name, code)
            return

        if self.may_be_a_sequence:
            for test_name in ('List', 'Tuple'):
2189 2190
                code.putln("if (!%s && Py%s_CheckExact(%s)) {" % (
                    self.iter_func_ptr, test_name, self.py_result()))
Stefan Behnel's avatar
Stefan Behnel committed
2191 2192 2193 2194 2195 2196 2197 2198 2199 2200 2201 2202 2203 2204 2205 2206 2207 2208
                self.generate_next_sequence_item(test_name, result_name, code)
                code.put("} else ")

        code.putln("{")
        code.putln(
            "%s = %s(%s);" % (
                result_name,
                self.iter_func_ptr,
                self.py_result()))
        code.putln("if (unlikely(!%s)) {" % result_name)
        code.putln("if (PyErr_Occurred()) {")
        code.putln("if (likely(PyErr_ExceptionMatches(PyExc_StopIteration))) PyErr_Clear();")
        code.putln("else %s" % code.error_goto(self.pos))
        code.putln("}")
        code.putln("break;")
        code.putln("}")
        code.put_gotref(result_name)
        code.putln("}")
William Stein's avatar
William Stein committed
2209

2210
    def free_temps(self, code):
2211 2212
        if self.counter_cname:
            code.funcstate.release_temp(self.counter_cname)
2213 2214 2215
        if self.iter_func_ptr:
            code.funcstate.release_temp(self.iter_func_ptr)
            self.iter_func_ptr = None
2216 2217
        if self.cpp_iterator_cname:
            code.funcstate.release_temp(self.cpp_iterator_cname)
2218
        ExprNode.free_temps(self, code)
William Stein's avatar
William Stein committed
2219 2220


2221
class NextNode(AtomicExprNode):
William Stein's avatar
William Stein committed
2222 2223 2224 2225 2226
    #  Used as part of for statement implementation.
    #  Implements result = iterator.next()
    #  Created during analyse_types phase.
    #  The iterator is not owned by this node.
    #
2227
    #  iterator   IteratorNode
2228

2229
    def __init__(self, iterator):
William Stein's avatar
William Stein committed
2230 2231
        self.pos = iterator.pos
        self.iterator = iterator
2232 2233 2234 2235

    def type_dependencies(self, env):
        return self.iterator.type_dependencies(env)

2236 2237 2238
    def infer_type(self, env, iterator_type = None):
        if iterator_type is None:
            iterator_type = self.iterator.infer_type(env)
2239
        if iterator_type.is_ptr or iterator_type.is_array:
2240
            return iterator_type.base_type
2241
        elif iterator_type.is_cpp_class:
2242
            item_type = env.lookup_operator_for_types(self.pos, "*", [iterator_type]).type.return_type
2243 2244
            if item_type.is_reference:
                item_type = item_type.ref_base_type
Robert Bradshaw's avatar
Robert Bradshaw committed
2245 2246
            if item_type.is_const:
                item_type = item_type.const_base_type
2247
            return item_type
2248
        else:
2249 2250
            # Avoid duplication of complicated logic.
            fake_index_node = IndexNode(self.pos,
2251
                                        base=self.iterator.sequence,
2252
                                        index=IntNode(self.pos, value='0'))
2253 2254 2255
            # TODO(vile hack): infer_type should be side-effect free
            if isinstance(self.iterator.sequence, SimpleCallNode):
                return py_object_type
2256
            return fake_index_node.infer_type(env)
2257 2258 2259

    def analyse_types(self, env):
        self.type = self.infer_type(env, self.iterator.type)
William Stein's avatar
William Stein committed
2260
        self.is_temp = 1
2261

William Stein's avatar
William Stein committed
2262
    def generate_result_code(self, code):
Stefan Behnel's avatar
Stefan Behnel committed
2263
        self.iterator.generate_iter_next_result_code(self.result(), code)
2264

William Stein's avatar
William Stein committed
2265

2266 2267 2268 2269 2270 2271 2272 2273 2274 2275 2276 2277 2278 2279 2280 2281 2282 2283 2284 2285 2286 2287 2288 2289 2290 2291 2292 2293 2294 2295 2296 2297 2298 2299
class WithExitCallNode(ExprNode):
    # The __exit__() call of a 'with' statement.  Used in both the
    # except and finally clauses.

    # with_stat  WithStatNode                the surrounding 'with' statement
    # args       TupleNode or ResultStatNode the exception info tuple

    subexprs = ['args']

    def analyse_types(self, env):
        self.args.analyse_types(env)
        self.type = PyrexTypes.c_bint_type
        self.is_temp = True

    def generate_result_code(self, code):
        if isinstance(self.args, TupleNode):
            # call only if it was not already called (and decref-cleared)
            code.putln("if (%s) {" % self.with_stat.exit_var)
        result_var = code.funcstate.allocate_temp(py_object_type, manage_ref=False)
        code.putln("%s = PyObject_Call(%s, %s, NULL);" % (
            result_var,
            self.with_stat.exit_var,
            self.args.result()))
        code.put_decref_clear(self.with_stat.exit_var, type=py_object_type)
        code.putln(code.error_goto_if_null(result_var, self.pos))
        code.put_gotref(result_var)
        code.putln("%s = __Pyx_PyObject_IsTrue(%s);" % (self.result(), result_var))
        code.put_decref_clear(result_var, type=py_object_type)
        code.putln(code.error_goto_if_neg(self.result(), self.pos))
        code.funcstate.release_temp(result_var)
        if isinstance(self.args, TupleNode):
            code.putln("}")


2300
class ExcValueNode(AtomicExprNode):
William Stein's avatar
William Stein committed
2301 2302 2303
    #  Node created during analyse_types phase
    #  of an ExceptClauseNode to fetch the current
    #  exception value.
2304

2305
    type = py_object_type
2306

2307
    def __init__(self, pos, env):
William Stein's avatar
William Stein committed
2308
        ExprNode.__init__(self, pos)
2309 2310

    def set_var(self, var):
2311
        self.var = var
2312

2313 2314 2315
    def calculate_result_code(self):
        return self.var

William Stein's avatar
William Stein committed
2316
    def generate_result_code(self, code):
2317
        pass
William Stein's avatar
William Stein committed
2318

2319 2320 2321
    def analyse_types(self, env):
        pass

William Stein's avatar
William Stein committed
2322

2323
class TempNode(ExprNode):
2324 2325 2326 2327 2328 2329 2330
    # Node created during analyse_types phase
    # of some nodes to hold a temporary value.
    #
    # Note: One must call "allocate" and "release" on
    # the node during code generation to get/release the temp.
    # This is because the temp result is often used outside of
    # the regular cycle.
2331 2332

    subexprs = []
2333

2334
    def __init__(self, pos, type, env=None):
William Stein's avatar
William Stein committed
2335 2336 2337 2338 2339
        ExprNode.__init__(self, pos)
        self.type = type
        if type.is_pyobject:
            self.result_ctype = py_object_type
        self.is_temp = 1
2340

2341 2342
    def analyse_types(self, env):
        return self.type
2343

2344 2345 2346
    def analyse_target_declaration(self, env):
        pass

William Stein's avatar
William Stein committed
2347 2348 2349
    def generate_result_code(self, code):
        pass

2350 2351 2352 2353 2354 2355 2356 2357 2358 2359 2360 2361 2362 2363 2364 2365 2366
    def allocate(self, code):
        self.temp_cname = code.funcstate.allocate_temp(self.type, manage_ref=True)

    def release(self, code):
        code.funcstate.release_temp(self.temp_cname)
        self.temp_cname = None

    def result(self):
        try:
            return self.temp_cname
        except:
            assert False, "Remember to call allocate/release on TempNode"
            raise

    # Do not participate in normal temp alloc/dealloc:
    def allocate_temp_result(self, code):
        pass
2367

2368 2369
    def release_temp_result(self, code):
        pass
William Stein's avatar
William Stein committed
2370 2371 2372

class PyTempNode(TempNode):
    #  TempNode holding a Python value.
2373

William Stein's avatar
William Stein committed
2374 2375 2376
    def __init__(self, pos, env):
        TempNode.__init__(self, pos, PyrexTypes.py_object_type, env)

2377 2378
class RawCNameExprNode(ExprNode):
    subexprs = []
2379

2380
    def __init__(self, pos, type=None, cname=None):
2381 2382
        self.pos = pos
        self.type = type
2383 2384
        if cname is not None:
            self.cname = cname
2385 2386 2387 2388 2389 2390 2391 2392 2393 2394 2395 2396 2397

    def analyse_types(self, env):
        return self.type

    def set_cname(self, cname):
        self.cname = cname

    def result(self):
        return self.cname

    def generate_result_code(self, code):
        pass

William Stein's avatar
William Stein committed
2398

Mark Florisson's avatar
Mark Florisson committed
2399 2400 2401 2402 2403 2404 2405 2406
#-------------------------------------------------------------------
#
#  Parallel nodes (cython.parallel.thread(savailable|id))
#
#-------------------------------------------------------------------

class ParallelThreadsAvailableNode(AtomicExprNode):
    """
Mark Florisson's avatar
Mark Florisson committed
2407 2408
    Note: this is disabled and not a valid directive at this moment

Mark Florisson's avatar
Mark Florisson committed
2409 2410 2411 2412 2413 2414 2415 2416 2417
    Implements cython.parallel.threadsavailable(). If we are called from the
    sequential part of the application, we need to call omp_get_max_threads(),
    and in the parallel part we can just call omp_get_num_threads()
    """

    type = PyrexTypes.c_int_type

    def analyse_types(self, env):
        self.is_temp = True
2418
        # env.add_include_file("omp.h")
Mark Florisson's avatar
Mark Florisson committed
2419 2420 2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442
        return self.type

    def generate_result_code(self, code):
        code.putln("#ifdef _OPENMP")
        code.putln("if (omp_in_parallel()) %s = omp_get_max_threads();" %
                                                            self.temp_code)
        code.putln("else %s = omp_get_num_threads();" % self.temp_code)
        code.putln("#else")
        code.putln("%s = 1;" % self.temp_code)
        code.putln("#endif")

    def result(self):
        return self.temp_code


class ParallelThreadIdNode(AtomicExprNode): #, Nodes.ParallelNode):
    """
    Implements cython.parallel.threadid()
    """

    type = PyrexTypes.c_int_type

    def analyse_types(self, env):
        self.is_temp = True
2443
        # env.add_include_file("omp.h")
Mark Florisson's avatar
Mark Florisson committed
2444 2445 2446 2447 2448 2449 2450 2451 2452 2453 2454 2455 2456
        return self.type

    def generate_result_code(self, code):
        code.putln("#ifdef _OPENMP")
        code.putln("%s = omp_get_thread_num();" % self.temp_code)
        code.putln("#else")
        code.putln("%s = 0;" % self.temp_code)
        code.putln("#endif")

    def result(self):
        return self.temp_code


William Stein's avatar
William Stein committed
2457 2458 2459 2460 2461 2462
#-------------------------------------------------------------------
#
#  Trailer nodes
#
#-------------------------------------------------------------------

2463
class IndexNode(ExprNode):
William Stein's avatar
William Stein committed
2464 2465 2466 2467
    #  Sequence indexing.
    #
    #  base     ExprNode
    #  index    ExprNode
2468 2469 2470 2471 2472 2473
    #  indices  [ExprNode]
    #  is_buffer_access boolean Whether this is a buffer access.
    #
    #  indices is used on buffer access, index on non-buffer access.
    #  The former contains a clean list of index parameters, the
    #  latter whatever Python object is needed for index access.
2474 2475 2476
    #
    #  is_fused_index boolean   Whether the index is used to specialize a
    #                           c(p)def function
2477

2478 2479 2480
    subexprs = ['base', 'index', 'indices']
    indices = None

2481 2482
    is_fused_index = False

2483 2484 2485 2486
    # Whether we're assigning to a buffer (in that case it needs to be
    # writable)
    writable_needed = False

2487 2488 2489
    # Whether we are indexing or slicing a memoryviewslice
    memslice_index = False
    memslice_slice = False
2490 2491
    is_memslice_copy = False
    memslice_ellipsis_noop = False
2492
    warned_untyped_idx = False
2493 2494
    # set by SingleAssignmentNode after analyse_types()
    is_memslice_scalar_assignment = False
2495

2496 2497 2498
    def __init__(self, pos, index, *args, **kw):
        ExprNode.__init__(self, pos, index=index, *args, **kw)
        self._index = index
2499 2500 2501 2502 2503

    def calculate_constant_result(self):
        self.constant_result = \
            self.base.constant_result[self.index.constant_result]

2504 2505 2506 2507 2508 2509 2510
    def compile_time_value(self, denv):
        base = self.base.compile_time_value(denv)
        index = self.index.compile_time_value(denv)
        try:
            return base[index]
        except Exception, e:
            self.compile_time_value_error(e)
2511

William Stein's avatar
William Stein committed
2512 2513
    def is_ephemeral(self):
        return self.base.is_ephemeral()
2514

2515
    def is_simple(self):
2516
        if self.is_buffer_access or self.memslice_index:
2517
            return False
2518 2519 2520
        elif self.memslice_slice:
            return True

2521 2522 2523 2524
        base = self.base
        return (base.is_simple() and self.index.is_simple()
                and base.type and (base.type.is_ptr or base.type.is_array))

William Stein's avatar
William Stein committed
2525 2526
    def analyse_target_declaration(self, env):
        pass
2527

2528 2529 2530
    def analyse_as_type(self, env):
        base_type = self.base.analyse_as_type(env)
        if base_type and not base_type.is_pyobject:
2531
            if base_type.is_cpp_class:
2532
                if isinstance(self.index, TupleNode):
2533 2534 2535 2536 2537
                    template_values = self.index.args
                else:
                    template_values = [self.index]
                import Nodes
                type_node = Nodes.TemplatedTypeNode(
2538 2539
                    pos = self.pos,
                    positional_args = template_values,
2540 2541 2542 2543
                    keyword_args = None)
                return type_node.analyse(env, base_type = base_type)
            else:
                return PyrexTypes.CArrayType(base_type, int(self.index.compile_time_value(env)))
2544
        return None
2545

Robert Bradshaw's avatar
Robert Bradshaw committed
2546
    def type_dependencies(self, env):
2547
        return self.base.type_dependencies(env) + self.index.type_dependencies(env)
2548

2549
    def infer_type(self, env):
2550 2551 2552 2553
        base_type = self.base.infer_type(env)
        if isinstance(self.index, SliceNode):
            # slicing!
            if base_type.is_string:
2554
                # sliced C strings must coerce to Python
2555
                return bytes_type
2556 2557 2558
            elif base_type in (unicode_type, bytes_type, str_type, list_type, tuple_type):
                # slicing these returns the same type
                return base_type
2559
            else:
2560 2561 2562
                # TODO: Handle buffers (hopefully without too much redundancy).
                return py_object_type

2563 2564
        index_type = self.index.infer_type(env)
        if index_type and index_type.is_int or isinstance(self.index, (IntNode, LongNode)):
2565 2566
            # indexing!
            if base_type is unicode_type:
2567 2568 2569
                # Py_UCS4 will automatically coerce to a unicode string
                # if required, so this is safe.  We only infer Py_UCS4
                # when the index is a C integer type.  Otherwise, we may
2570 2571 2572 2573
                # need to use normal Python item access, in which case
                # it's faster to return the one-char unicode string than
                # to receive it, throw it away, and potentially rebuild it
                # on a subsequent PyObject coercion.
2574
                return PyrexTypes.c_py_ucs4_type
2575 2576 2577
            elif base_type is str_type:
                # always returns str - Py2: bytes, Py3: unicode
                return base_type
2578 2579 2580 2581 2582 2583
            elif isinstance(self.base, BytesNode):
                #if env.global_scope().context.language_level >= 3:
                #    # infering 'char' can be made to work in Python 3 mode
                #    return PyrexTypes.c_char_type
                # Py2/3 return different types on indexing bytes objects
                return py_object_type
2584 2585
            elif base_type.is_ptr or base_type.is_array:
                return base_type.base_type
2586

2587 2588 2589 2590 2591 2592 2593 2594 2595 2596
        if base_type.is_cpp_class:
            class FakeOperand:
                def __init__(self, **kwds):
                    self.__dict__.update(kwds)
            operands = [
                FakeOperand(pos=self.pos, type=base_type),
                FakeOperand(pos=self.pos, type=index_type),
            ]
            index_func = env.lookup_operator('[]', operands)
            if index_func is not None:
2597
                return index_func.type.return_type
2598

2599
        # may be slicing or indexing, we don't know
2600 2601
        if base_type in (unicode_type, str_type):
            # these types always returns their own type on Python indexing/slicing
2602
            return base_type
2603 2604 2605
        else:
            # TODO: Handle buffers (hopefully without too much redundancy).
            return py_object_type
2606

William Stein's avatar
William Stein committed
2607
    def analyse_types(self, env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2608
        self.analyse_base_and_index_types(env, getting = 1)
2609

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
2610 2611
    def analyse_target_types(self, env):
        self.analyse_base_and_index_types(env, setting = 1)
Robert Bradshaw's avatar
Robert Bradshaw committed
2612 2613
        if self.type.is_const:
            error(self.pos, "Assignment to const dereference")
2614 2615
        if not self.is_lvalue():
            error(self.pos, "Assignment to non-lvalue of type '%s'" % self.type)
2616

2617
    def analyse_base_and_index_types(self, env, getting = 0, setting = 0, analyse_base = True):
2618 2619 2620
        # Note: This might be cleaned up by having IndexNode
        # parsed in a saner way and only construct the tuple if
        # needed.
2621 2622 2623 2624

        # Note that this function must leave IndexNode in a cloneable state.
        # For buffers, self.index is packed out on the initial analysis, and
        # when cloning self.indices is copied.
2625 2626
        self.is_buffer_access = False

2627
        # a[...] = b
2628
        self.is_memslice_copy = False
2629 2630 2631 2632
        # incomplete indexing, Ellipsis indexing or slicing
        self.memslice_slice = False
        # integer indexing
        self.memslice_index = False
2633

2634 2635 2636
        if analyse_base:
            self.base.analyse_types(env)

2637 2638 2639 2640 2641
        if self.base.type.is_error:
            # Do not visit child tree if base is undeclared to avoid confusing
            # error messages
            self.type = PyrexTypes.error_type
            return
2642

2643
        is_slice = isinstance(self.index, SliceNode)
2644

2645
        # Potentially overflowing index value.
2646
        if not is_slice and isinstance(self.index, IntNode) and Utils.long_literal(self.index.value):
2647
            self.index = self.index.coerce_to_pyobject(env)
2648

2649 2650
        is_memslice = self.base.type.is_memoryviewslice

2651
        # Handle the case where base is a literal char* (and we expect a string, not an int)
2652
        if not is_memslice and (isinstance(self.base, BytesNode) or is_slice):
Robert Bradshaw's avatar
Robert Bradshaw committed
2653
            if self.base.type.is_string or not (self.base.type.is_ptr or self.base.type.is_array):
2654
                self.base = self.base.coerce_to_pyobject(env)
2655 2656 2657

        skip_child_analysis = False
        buffer_access = False
Mark Florisson's avatar
Mark Florisson committed
2658

2659 2660 2661 2662 2663 2664 2665
        if self.indices:
            indices = self.indices
        elif isinstance(self.index, TupleNode):
            indices = self.index.args
        else:
            indices = [self.index]

2666
        if (is_memslice and not self.indices and
Mark Florisson's avatar
Mark Florisson committed
2667
                isinstance(self.index, EllipsisNode)):
2668
            # Memoryviewslice copying
2669
            self.is_memslice_copy = True
2670 2671 2672 2673 2674 2675

        elif is_memslice:
            # memoryviewslice indexing or slicing
            import MemoryView

            skip_child_analysis = True
2676
            newaxes = [newaxis for newaxis in indices if newaxis.is_none]
2677
            have_slices, indices = MemoryView.unellipsify(indices,
2678
                                                          newaxes,
2679
                                                          self.base.type.ndim)
2680 2681 2682

            self.memslice_index = (not newaxes and
                                   len(indices) == self.base.type.ndim)
2683 2684 2685 2686 2687
            axes = []

            index_type = PyrexTypes.c_py_ssize_t_type
            new_indices = []

2688
            if len(indices) - len(newaxes) > self.base.type.ndim:
2689 2690 2691 2692 2693
                self.type = error_type
                return error(indices[self.base.type.ndim].pos,
                             "Too many indices specified for type %s" %
                                                        self.base.type)

2694
            axis_idx = 0
2695 2696
            for i, index in enumerate(indices[:]):
                index.analyse_types(env)
2697 2698 2699 2700
                if not index.is_none:
                    access, packing = self.base.type.axes[axis_idx]
                    axis_idx += 1

2701 2702
                if isinstance(index, SliceNode):
                    self.memslice_slice = True
2703 2704
                    if index.step.is_none:
                        axes.append((access, packing))
2705 2706
                    else:
                        axes.append((access, 'strided'))
2707 2708 2709 2710 2711 2712

                    # Coerce start, stop and step to temps of the right type
                    for attr in ('start', 'stop', 'step'):
                        value = getattr(index, attr)
                        if not value.is_none:
                            value = value.coerce_to(index_type, env)
2713
                            #value = value.coerce_to_temp(env)
2714 2715 2716
                            setattr(index, attr, value)
                            new_indices.append(value)

2717 2718 2719 2720 2721
                elif index.is_none:
                    self.memslice_slice = True
                    new_indices.append(index)
                    axes.append(('direct', 'strided'))

2722 2723 2724 2725 2726
                elif index.type.is_int or index.type.is_pyobject:
                    if index.type.is_pyobject and not self.warned_untyped_idx:
                        warning(index.pos, "Index should be typed for more "
                                           "efficient access", level=2)
                        IndexNode.warned_untyped_idx = True
2727

2728
                    self.memslice_index = True
2729
                    index = index.coerce_to(index_type, env)
2730 2731 2732
                    indices[i] = index
                    new_indices.append(index)

2733
                else:
2734 2735
                    self.type = error_type
                    return error(index.pos, "Invalid index for memoryview specified")
2736

2737 2738
            self.memslice_index = self.memslice_index and not self.memslice_slice
            self.original_indices = indices
2739 2740
            # All indices with all start/stop/step for slices.
            # We need to keep this around
2741
            self.indices = new_indices
2742 2743
            self.env = env

2744 2745
        elif self.base.type.is_buffer:
            # Buffer indexing
2746
            if len(indices) == self.base.type.ndim:
2747 2748 2749 2750 2751 2752
                buffer_access = True
                skip_child_analysis = True
                for x in indices:
                    x.analyse_types(env)
                    if not x.type.is_int:
                        buffer_access = False
2753

2754
            if buffer_access and not self.base.type.is_memoryviewslice:
Robert Bradshaw's avatar
Robert Bradshaw committed
2755
                assert hasattr(self.base, "entry") # Must be a NameNode-like node
2756

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2757 2758 2759
        # On cloning, indices is cloned. Otherwise, unpack index into indices
        assert not (buffer_access and isinstance(self.index, CloneNode))

2760 2761
        self.nogil = env.nogil

2762
        if buffer_access or self.memslice_index:
2763 2764 2765
            #if self.base.type.is_memoryviewslice and not self.base.is_name:
            #    self.base = self.base.coerce_to_temp(env)
            self.base = self.base.coerce_to_simple(env)
2766

2767
            self.indices = indices
2768
            self.index = None
2769 2770
            self.type = self.base.type.dtype
            self.is_buffer_access = True
2771
            self.buffer_type = self.base.type #self.base.entry.type
2772 2773

            if getting and self.type.is_pyobject:
2774
                self.is_temp = True
2775 2776

            if setting and self.base.type.is_memoryviewslice:
2777
                self.base.type.writable_needed = True
2778
            elif setting:
2779 2780 2781
                if not self.base.entry.type.writable:
                    error(self.pos, "Writing to readonly buffer")
                else:
2782
                    self.writable_needed = True
2783
                    if self.base.type.is_buffer:
2784
                        self.base.entry.buffer_aux.writable_needed = True
2785

2786
        elif self.is_memslice_copy:
Mark Florisson's avatar
Mark Florisson committed
2787 2788
            self.type = self.base.type
            if getting:
2789 2790 2791
                self.memslice_ellipsis_noop = True
            else:
                self.memslice_broadcast = True
2792

2793
        elif self.memslice_slice:
2794
            self.index = None
2795
            self.is_temp = True
2796
            self.use_managed_ref = True
2797 2798 2799 2800 2801

            if not MemoryView.validate_axes(self.pos, axes):
                self.type = error_type
                return

2802 2803
            self.type = PyrexTypes.MemoryViewSliceType(
                            self.base.type.dtype, axes)
2804 2805 2806 2807 2808 2809

            if (self.base.type.is_memoryviewslice and not
                    self.base.is_name and not
                    self.base.result_in_temp()):
                self.base = self.base.coerce_to_temp(env)

2810 2811
            if setting:
                self.memslice_broadcast = True
2812

2813
        else:
2814
            base_type = self.base.type
2815 2816 2817 2818 2819 2820 2821 2822 2823

            fused_index_operation = base_type.is_cfunction and base_type.is_fused
            if not fused_index_operation:
                if isinstance(self.index, TupleNode):
                    self.index.analyse_types(env, skip_children=skip_child_analysis)
                elif not skip_child_analysis:
                    self.index.analyse_types(env)
                self.original_index_type = self.index.type

Stefan Behnel's avatar
Stefan Behnel committed
2824 2825
            if base_type.is_unicode_char:
                # we infer Py_UNICODE/Py_UCS4 for unicode strings in some
2826 2827 2828 2829 2830 2831 2832
                # cases, but indexing must still work for them
                if self.index.constant_result in (0, -1):
                    # FIXME: we know that this node is redundant -
                    # currently, this needs to get handled in Optimize.py
                    pass
                self.base = self.base.coerce_to_pyobject(env)
                base_type = self.base.type
2833
            if base_type.is_pyobject:
2834
                if self.index.type.is_int:
2835
                    if (not setting
2836
                        and (base_type in (list_type, tuple_type))
2837 2838
                        and (not self.index.type.signed
                             or not env.directives['wraparound']
2839 2840
                             or (isinstance(self.index, IntNode) and
                                 self.index.has_constant_result() and self.index.constant_result >= 0))
2841 2842 2843 2844
                        and not env.directives['boundscheck']):
                        self.is_temp = 0
                    else:
                        self.is_temp = 1
2845 2846 2847
                    self.index = self.index.coerce_to(PyrexTypes.c_py_ssize_t_type, env).coerce_to_simple(env)
                else:
                    self.index = self.index.coerce_to_pyobject(env)
2848
                    self.is_temp = 1
2849
                if self.index.type.is_int and base_type is unicode_type:
Stefan Behnel's avatar
Stefan Behnel committed
2850
                    # Py_UNICODE/Py_UCS4 will automatically coerce to a unicode string
2851
                    # if required, so this is fast and safe
2852
                    self.type = PyrexTypes.c_py_ucs4_type
2853 2854
                elif is_slice and base_type in (bytes_type, str_type, unicode_type, list_type, tuple_type):
                    self.type = base_type
2855
                else:
2856 2857 2858
                    if base_type in (list_type, tuple_type, dict_type):
                        # do the None check explicitly (not in a helper) to allow optimising it away
                        self.base = self.base.as_none_safe_node("'NoneType' object is not subscriptable")
2859
                    self.type = py_object_type
William Stein's avatar
William Stein committed
2860
            else:
2861 2862
                if base_type.is_ptr or base_type.is_array:
                    self.type = base_type.base_type
2863 2864 2865
                    if is_slice:
                        self.type = base_type
                    elif self.index.type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
2866 2867
                        self.index = self.index.coerce_to(
                            PyrexTypes.c_py_ssize_t_type, env)
2868
                    elif not self.index.type.is_int:
Robert Bradshaw's avatar
Robert Bradshaw committed
2869 2870 2871
                        error(self.pos,
                            "Invalid index type '%s'" %
                                self.index.type)
2872
                elif base_type.is_cpp_class:
2873
                    function = env.lookup_operator("[]", [self.base, self.index])
Robert Bradshaw's avatar
Robert Bradshaw committed
2874
                    if function is None:
2875
                        error(self.pos, "Indexing '%s' not supported for index type '%s'" % (base_type, self.index.type))
Robert Bradshaw's avatar
Robert Bradshaw committed
2876 2877 2878 2879 2880 2881 2882 2883 2884
                        self.type = PyrexTypes.error_type
                        self.result_code = "<error>"
                        return
                    func_type = function.type
                    if func_type.is_ptr:
                        func_type = func_type.base_type
                    self.index = self.index.coerce_to(func_type.args[0].type, env)
                    self.type = func_type.return_type
                    if setting and not func_type.return_type.is_reference:
Robert Bradshaw's avatar
Robert Bradshaw committed
2885
                        error(self.pos, "Can't set non-reference result '%s'" % self.type)
2886 2887
                elif fused_index_operation:
                    self.parse_indexed_fused_cdef(env)
2888 2889 2890
                else:
                    error(self.pos,
                        "Attempting to index non-array type '%s'" %
2891
                            base_type)
2892
                    self.type = PyrexTypes.error_type
Stefan Behnel's avatar
Stefan Behnel committed
2893

2894 2895 2896 2897 2898 2899 2900 2901 2902 2903 2904 2905 2906 2907 2908 2909 2910 2911
        self.wrap_in_nonecheck_node(env, getting)

    def wrap_in_nonecheck_node(self, env, getting):
        if not env.directives['nonecheck'] or not self.base.may_be_none():
            return

        if self.base.type.is_memoryviewslice:
            if self.is_memslice_copy and not getting:
                msg = "Cannot assign to None memoryview slice"
            elif self.memslice_slice:
                msg = "Cannot slice None memoryview slice"
            else:
                msg = "Cannot index None memoryview slice"
        else:
            msg = "'NoneType' object is not subscriptable"

        self.base = self.base.as_none_safe_node(msg)

2912 2913 2914 2915 2916 2917 2918 2919 2920
    def parse_indexed_fused_cdef(self, env):
        """
        Interpret fused_cdef_func[specific_type1, ...]

        Note that if this method is called, we are an indexed cdef function
        with fused argument types, and this IndexNode will be replaced by the
        NameNode with specific entry just after analysis of expressions by
        AnalyseExpressionsTransform.
        """
2921
        self.type = PyrexTypes.error_type
2922

2923 2924
        self.is_fused_index = True

Mark Florisson's avatar
Mark Florisson committed
2925
        base_type = self.base.type
2926 2927 2928
        specific_types = []
        positions = []

2929
        if self.index.is_name or self.index.is_attribute:
2930 2931 2932 2933 2934
            positions.append(self.index.pos)
            specific_types.append(self.index.analyse_as_type(env))
        elif isinstance(self.index, TupleNode):
            for arg in self.index.args:
                positions.append(arg.pos)
2935 2936
                specific_type = arg.analyse_as_type(env)
                specific_types.append(specific_type)
2937
        else:
2938
            specific_types = [False]
2939

2940 2941 2942 2943
        if not Utils.all(specific_types):
            self.index.analyse_types(env)

            if not self.base.entry.as_variable:
2944
                error(self.pos, "Can only index fused functions with types")
2945 2946
            else:
                # A cpdef function indexed with Python objects
2947 2948
                self.base.entry = self.entry = self.base.entry.as_variable
                self.base.type = self.type = self.entry.type
2949

2950 2951 2952 2953 2954 2955
                self.base.is_temp = True
                self.is_temp = True

                self.entry.used = True

            self.is_fused_index = False
2956 2957
            return

Mark Florisson's avatar
Mark Florisson committed
2958 2959 2960
        for i, type in enumerate(specific_types):
            specific_types[i] = type.specialize_fused(env)

2961 2962
        fused_types = base_type.get_fused_types()
        if len(specific_types) > len(fused_types):
2963 2964 2965 2966 2967
            return error(self.pos, "Too many types specified")
        elif len(specific_types) < len(fused_types):
            t = fused_types[len(specific_types)]
            return error(self.pos, "Not enough types specified to specialize "
                                   "the function, %s is still fused" % t)
2968 2969 2970 2971 2972 2973 2974

        # See if our index types form valid specializations
        for pos, specific_type, fused_type in zip(positions,
                                                  specific_types,
                                                  fused_types):
            if not Utils.any([specific_type.same_as(t)
                                  for t in fused_type.types]):
2975
                return error(pos, "Type not in fused type")
2976 2977 2978 2979 2980 2981 2982

            if specific_type is None or specific_type.is_error:
                return

        fused_to_specific = dict(zip(fused_types, specific_types))
        type = base_type.specialize(fused_to_specific)

2983 2984 2985 2986 2987
        if type.is_fused:
            # Only partially specific, this is invalid
            error(self.pos,
                  "Index operation makes function only partially specific")
        else:
2988
            # Fully specific, find the signature with the specialized entry
2989
            for signature in self.base.type.get_all_specialized_function_types():
2990 2991
                if type.same_as(signature):
                    self.type = signature
Mark Florisson's avatar
Mark Florisson committed
2992 2993 2994 2995 2996

                    if self.base.is_attribute:
                        # Pretend to be a normal attribute, for cdef extension
                        # methods
                        self.entry = signature.entry
2997
                        self.is_attribute = True
Mark Florisson's avatar
Mark Florisson committed
2998
                        self.obj = self.base.obj
2999 3000 3001 3002

                    self.type.entry.used = True
                    self.base.type = signature
                    self.base.entry = signature.entry
Mark Florisson's avatar
Mark Florisson committed
3003

3004 3005
                    break
            else:
Mark Florisson's avatar
Mark Florisson committed
3006 3007
                # This is a bug
                raise InternalError("Couldn't find the right signature")
3008

3009 3010
    gil_message = "Indexing Python object"

3011
    def nogil_check(self, env):
3012 3013
        if self.is_buffer_access or self.memslice_index or self.memslice_slice:
            if not self.memslice_slice and env.directives['boundscheck']:
3014 3015 3016 3017 3018
                # error(self.pos, "Cannot check buffer index bounds without gil; "
                #                 "use boundscheck(False) directive")
                warning(self.pos, "Use boundscheck(False) for faster access",
                        level=1)
            if self.type.is_pyobject:
3019 3020
                error(self.pos, "Cannot access buffer with object dtype without gil")
                return
3021
        super(IndexNode, self).nogil_check(env)
3022 3023


William Stein's avatar
William Stein committed
3024
    def check_const_addr(self):
3025
        return self.base.check_const_addr() and self.index.check_const()
3026

William Stein's avatar
William Stein committed
3027
    def is_lvalue(self):
3028 3029 3030 3031 3032
        base_type = self.base.type
        if self.type.is_ptr or self.type.is_array:
            return not base_type.base_type.is_array
        else:
            return True
Dag Sverre Seljebotn's avatar
merge  
Dag Sverre Seljebotn committed
3033

William Stein's avatar
William Stein committed
3034
    def calculate_result_code(self):
3035
        if self.is_buffer_access:
3036
            return "(*%s)" % self.buffer_ptr_code
3037 3038
        elif self.is_memslice_copy:
            return self.base.result()
3039 3040 3041 3042
        elif self.base.type is list_type:
            return "PyList_GET_ITEM(%s, %s)" % (self.base.result(), self.index.result())
        elif self.base.type is tuple_type:
            return "PyTuple_GET_ITEM(%s, %s)" % (self.base.result(), self.index.result())
3043 3044
        elif (self.type.is_ptr or self.type.is_array) and self.type == self.base.type:
            error(self.pos, "Invalid use of pointer slice")
3045 3046
        else:
            return "(%s[%s])" % (
3047
                self.base.result(), self.index.result())
3048

3049
    def extra_index_params(self):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3050 3051
        if self.index.type.is_int:
            if self.original_index_type.signed:
3052
                size_adjustment = ""
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3053
            else:
3054 3055
                size_adjustment = "+1"
            return ", sizeof(%s)%s, %s" % (self.original_index_type.declaration_code(""), size_adjustment, self.original_index_type.to_py_function)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3056 3057
        else:
            return ""
3058 3059 3060

    def generate_subexpr_evaluation_code(self, code):
        self.base.generate_evaluation_code(code)
3061
        if self.indices is None:
3062 3063
            self.index.generate_evaluation_code(code)
        else:
3064 3065
            for i in self.indices:
                i.generate_evaluation_code(code)
3066

3067 3068
    def generate_subexpr_disposal_code(self, code):
        self.base.generate_disposal_code(code)
3069
        if self.indices is None:
3070 3071
            self.index.generate_disposal_code(code)
        else:
3072 3073
            for i in self.indices:
                i.generate_disposal_code(code)
3074

3075 3076
    def free_subexpr_temps(self, code):
        self.base.free_temps(code)
3077
        if self.indices is None:
3078 3079 3080 3081 3082
            self.index.free_temps(code)
        else:
            for i in self.indices:
                i.free_temps(code)

William Stein's avatar
William Stein committed
3083
    def generate_result_code(self, code):
3084
        if self.is_buffer_access or self.memslice_index:
3085
            buffer_entry, self.buffer_ptr_code = self.buffer_lookup_code(code)
3086 3087 3088
            if self.type.is_pyobject:
                # is_temp is True, so must pull out value and incref it.
                code.putln("%s = *%s;" % (self.result(), self.buffer_ptr_code))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
3089
                code.putln("__Pyx_INCREF((PyObject*)%s);" % self.result())
3090 3091 3092 3093

        elif self.memslice_slice:
            self.put_memoryviewslice_slice_code(code)

3094 3095 3096 3097 3098 3099 3100 3101 3102 3103
        elif self.is_temp:
            if self.type.is_pyobject:
                if self.index.type.is_int:
                    index_code = self.index.result()
                    if self.base.type is list_type:
                        function = "__Pyx_GetItemInt_List"
                    elif self.base.type is tuple_type:
                        function = "__Pyx_GetItemInt_Tuple"
                    else:
                        function = "__Pyx_GetItemInt"
3104 3105
                    code.globalstate.use_utility_code(
                        TempitaUtilityCode.load_cached("GetItemInt", "ObjectHandling.c"))
3106
                else:
3107 3108 3109
                    index_code = self.index.py_result()
                    if self.base.type is dict_type:
                        function = "__Pyx_PyDict_GetItem"
3110 3111
                        code.globalstate.use_utility_code(
                            UtilityCode.load_cached("DictGetItem", "ObjectHandling.c"))
3112 3113 3114 3115 3116 3117 3118 3119 3120 3121 3122 3123
                    else:
                        function = "PyObject_GetItem"
                code.putln(
                    "%s = %s(%s, %s%s); if (!%s) %s" % (
                        self.result(),
                        function,
                        self.base.py_result(),
                        index_code,
                        self.extra_index_params(),
                        self.result(),
                        code.error_goto(self.pos)))
                code.put_gotref(self.py_result())
Stefan Behnel's avatar
Stefan Behnel committed
3124
            elif self.type.is_unicode_char and self.base.type is unicode_type:
3125 3126 3127
                assert self.index.type.is_int
                index_code = self.index.result()
                function = "__Pyx_GetItemInt_Unicode"
3128 3129
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("GetItemIntUnicode", "StringTools.c"))
3130
                code.putln(
3131
                    "%s = %s(%s, %s%s); if (unlikely(%s == (Py_UCS4)-1)) %s;" % (
3132 3133 3134 3135 3136 3137 3138
                        self.result(),
                        function,
                        self.base.py_result(),
                        index_code,
                        self.extra_index_params(),
                        self.result(),
                        code.error_goto(self.pos)))
3139

Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3140 3141 3142
    def generate_setitem_code(self, value_code, code):
        if self.index.type.is_int:
            function = "__Pyx_SetItemInt"
3143
            index_code = self.index.result()
3144 3145
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("SetItemInt", "ObjectHandling.c"))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3146 3147
        else:
            index_code = self.index.py_result()
3148 3149
            if self.base.type is dict_type:
                function = "PyDict_SetItem"
Craig Citro's avatar
Craig Citro committed
3150
            # It would seem that we could specialized lists/tuples, but that
3151
            # shouldn't happen here.
Stefan Behnel's avatar
Stefan Behnel committed
3152 3153 3154 3155 3156
            # Both PyList_SetItem() and PyTuple_SetItem() take a Py_ssize_t as
            # index instead of an object, and bad conversion here would give
            # the wrong exception. Also, tuples are supposed to be immutable,
            # and raise a TypeError when trying to set their entries
            # (PyTuple_SetItem() is for creating new tuples from scratch).
3157 3158
            else:
                function = "PyObject_SetItem"
3159
        code.putln(
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3160 3161
            "if (%s(%s, %s, %s%s) < 0) %s" % (
                function,
3162
                self.base.py_result(),
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3163 3164
                index_code,
                value_code,
3165
                self.extra_index_params(),
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3166
                code.error_goto(self.pos)))
3167 3168 3169

    def generate_buffer_setitem_code(self, rhs, code, op=""):
        # Used from generate_assignment_code and InPlaceAssignmentNode
3170 3171
        buffer_entry, ptrexpr = self.buffer_lookup_code(code)

3172 3173 3174
        if self.buffer_type.dtype.is_pyobject:
            # Must manage refcounts. Decref what is already there
            # and incref what we put in.
3175 3176
            ptr = code.funcstate.allocate_temp(buffer_entry.buf_ptr_type,
                                               manage_ref=False)
3177
            rhs_code = rhs.result()
3178
            code.putln("%s = %s;" % (ptr, ptrexpr))
3179
            code.put_gotref("*%s" % ptr)
3180 3181
            code.putln("__Pyx_INCREF(%s); __Pyx_DECREF(*%s);" % (
                rhs_code, ptr))
3182
            code.putln("*%s %s= %s;" % (ptr, op, rhs_code))
3183
            code.put_giveref("*%s" % ptr)
3184
            code.funcstate.release_temp(ptr)
3185
        else:
3186
            # Simple case
3187
            code.putln("*%s %s= %s;" % (ptrexpr, op, rhs.result()))
3188

William Stein's avatar
William Stein committed
3189
    def generate_assignment_code(self, rhs, code):
3190 3191 3192 3193 3194 3195 3196
        generate_evaluation_code = (self.is_memslice_scalar_assignment or
                                    self.memslice_slice)
        if generate_evaluation_code:
            self.generate_evaluation_code(code)
        else:
            self.generate_subexpr_evaluation_code(code)

3197
        if self.is_buffer_access or self.memslice_index:
3198
            self.generate_buffer_setitem_code(rhs, code)
3199 3200
        elif self.is_memslice_scalar_assignment:
            self.generate_memoryviewslice_assign_scalar_code(rhs, code)
3201
        elif self.memslice_slice or self.is_memslice_copy:
3202
            self.generate_memoryviewslice_setslice_code(rhs, code)
3203
        elif self.type.is_pyobject:
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3204
            self.generate_setitem_code(rhs.py_result(), code)
William Stein's avatar
William Stein committed
3205 3206 3207
        else:
            code.putln(
                "%s = %s;" % (
3208
                    self.result(), rhs.result()))
3209 3210 3211 3212 3213 3214 3215

        if generate_evaluation_code:
            self.generate_disposal_code(code)
        else:
            self.generate_subexpr_disposal_code(code)
            self.free_subexpr_temps(code)

William Stein's avatar
William Stein committed
3216
        rhs.generate_disposal_code(code)
3217
        rhs.free_temps(code)
3218

William Stein's avatar
William Stein committed
3219 3220
    def generate_deletion_code(self, code):
        self.generate_subexpr_evaluation_code(code)
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3221 3222
        #if self.type.is_pyobject:
        if self.index.type.is_int:
3223
            function = "__Pyx_DelItemInt"
3224
            index_code = self.index.result()
3225 3226
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("DelItemInt", "ObjectHandling.c"))
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3227 3228
        else:
            index_code = self.index.py_result()
3229 3230 3231 3232
            if self.base.type is dict_type:
                function = "PyDict_DelItem"
            else:
                function = "PyObject_DelItem"
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3233
        code.putln(
3234
            "if (%s(%s, %s%s) < 0) %s" % (
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3235
                function,
William Stein's avatar
William Stein committed
3236
                self.base.py_result(),
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3237
                index_code,
3238
                self.extra_index_params(),
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
3239
                code.error_goto(self.pos)))
William Stein's avatar
William Stein committed
3240
        self.generate_subexpr_disposal_code(code)
3241
        self.free_subexpr_temps(code)
3242

3243 3244 3245
    def buffer_entry(self):
        import Buffer, MemoryView

3246 3247 3248 3249 3250 3251
        base = self.base
        if self.base.is_nonecheck:
            base = base.arg

        if base.is_name:
            entry = base.entry
3252
        else:
3253
            # SimpleCallNode is_simple is not consistent with coerce_to_simple
3254 3255
            assert base.is_simple() or base.is_temp
            cname = base.result()
3256 3257 3258 3259 3260 3261 3262 3263 3264
            entry = Symtab.Entry(cname, cname, self.base.type, self.base.pos)

        if entry.type.is_buffer:
            buffer_entry = Buffer.BufferEntry(entry)
        else:
            buffer_entry = MemoryView.MemoryViewSliceBufferEntry(entry)

        return buffer_entry

3265
    def buffer_lookup_code(self, code):
3266
        "ndarray[1, 2, 3] and memslice[1, 2, 3]"
3267
        # Assign indices to temps
3268 3269 3270
        index_temps = [code.funcstate.allocate_temp(i.type, manage_ref=False)
                           for i in self.indices]

3271
        for temp, index in zip(index_temps, self.indices):
3272
            code.putln("%s = %s;" % (temp, index.result()))
3273

3274
        # Generate buffer access code using these temps
3275
        import Buffer, MemoryView
3276

3277
        buffer_entry = self.buffer_entry()
3278

3279
        if buffer_entry.type.is_buffer:
3280
            negative_indices = buffer_entry.type.negative_indices
3281 3282 3283
        else:
            negative_indices = Buffer.buffer_defaults['negative_indices']

3284 3285 3286 3287 3288 3289
        return buffer_entry, Buffer.put_buffer_lookup_code(
               entry=buffer_entry,
               index_signeds=[i.type.signed for i in self.indices],
               index_cnames=index_temps,
               directives=code.globalstate.directives,
               pos=self.pos, code=code,
3290 3291
               negative_indices=negative_indices,
               in_nogil_context=self.in_nogil_context)
William Stein's avatar
William Stein committed
3292

3293
    def put_memoryviewslice_slice_code(self, code):
3294
        "memslice[:]"
3295
        buffer_entry = self.buffer_entry()
3296
        have_gil = not self.in_nogil_context
3297

3298 3299 3300 3301 3302 3303
        if sys.version_info < (3,):
            def next_(it):
                return it.next()
        else:
            next_ = next

3304 3305 3306 3307 3308 3309 3310
        have_slices = False
        it = iter(self.indices)
        for index in self.original_indices:
            is_slice = isinstance(index, SliceNode)
            have_slices = have_slices or is_slice
            if is_slice:
                if not index.start.is_none:
3311
                    index.start = next_(it)
3312
                if not index.stop.is_none:
3313
                    index.stop = next_(it)
3314
                if not index.step.is_none:
3315
                    index.step = next_(it)
3316
            else:
3317
                next_(it)
3318 3319 3320 3321

        assert not list(it)

        buffer_entry.generate_buffer_slice_code(code, self.original_indices,
3322
                                                self.result(),
3323 3324
                                                have_gil=have_gil,
                                                have_slices=have_slices)
William Stein's avatar
William Stein committed
3325

3326
    def generate_memoryviewslice_setslice_code(self, rhs, code):
3327
        "memslice1[...] = memslice2 or memslice1[:] = memslice2"
3328 3329 3330
        import MemoryView
        MemoryView.copy_broadcast_memview_src_to_dst(rhs, self, code)

3331 3332 3333 3334 3335
    def generate_memoryviewslice_assign_scalar_code(self, rhs, code):
        "memslice1[...] = 0.0 or memslice1[:] = 0.0"
        import MemoryView
        MemoryView.assign_scalar(self, rhs, code)

3336

3337
class SliceIndexNode(ExprNode):
William Stein's avatar
William Stein committed
3338 3339 3340 3341 3342
    #  2-element slice indexing
    #
    #  base      ExprNode
    #  start     ExprNode or None
    #  stop      ExprNode or None
3343

William Stein's avatar
William Stein committed
3344
    subexprs = ['base', 'start', 'stop']
3345

3346 3347
    def infer_type(self, env):
        base_type = self.base.infer_type(env)
3348
        if base_type.is_string or base_type.is_cpp_class:
3349 3350 3351 3352
            return bytes_type
        elif base_type in (bytes_type, str_type, unicode_type,
                           list_type, tuple_type):
            return base_type
3353 3354
        elif base_type.is_ptr or base_type.is_array:
            return PyrexTypes.c_array_type(base_type.base_type, None)
3355 3356
        return py_object_type

3357 3358 3359 3360
    def calculate_constant_result(self):
        self.constant_result = self.base.constant_result[
            self.start.constant_result : self.stop.constant_result]

3361 3362
    def compile_time_value(self, denv):
        base = self.base.compile_time_value(denv)
3363 3364 3365 3366 3367 3368 3369 3370
        if self.start is None:
            start = 0
        else:
            start = self.start.compile_time_value(denv)
        if self.stop is None:
            stop = None
        else:
            stop = self.stop.compile_time_value(denv)
3371 3372 3373 3374
        try:
            return base[start:stop]
        except Exception, e:
            self.compile_time_value_error(e)
3375

William Stein's avatar
William Stein committed
3376 3377
    def analyse_target_declaration(self, env):
        pass
3378

3379
    def analyse_target_types(self, env):
3380
        self.analyse_types(env, getting=False)
3381
        # when assigning, we must accept any Python type
3382 3383
        if self.type.is_pyobject:
            self.type = py_object_type
William Stein's avatar
William Stein committed
3384

3385
    def analyse_types(self, env, getting=True):
William Stein's avatar
William Stein committed
3386
        self.base.analyse_types(env)
3387 3388 3389 3390 3391 3392 3393 3394 3395 3396 3397 3398 3399 3400 3401 3402 3403 3404 3405 3406

        if self.base.type.is_memoryviewslice:
            # Gross hack here! But we do not know the type until this point,
            # and we cannot create and return a new node. So we change the
            # type...
            none_node = NoneNode(self.pos)
            index = SliceNode(self.pos,
                              start=self.start or none_node,
                              stop=self.stop or none_node,
                              step=none_node)
            del self.start
            del self.stop
            self.index = index
            self.__class__ = IndexNode
            self.analyse_base_and_index_types(env,
                                              getting=getting,
                                              setting=not getting,
                                              analyse_base=False)
            return

William Stein's avatar
William Stein committed
3407 3408 3409 3410
        if self.start:
            self.start.analyse_types(env)
        if self.stop:
            self.stop.analyse_types(env)
3411
        base_type = self.base.type
3412
        if base_type.is_string or base_type.is_cpp_string:
3413
            self.type = bytes_type
3414 3415 3416
        elif base_type.is_ptr:
            self.type = base_type
        elif base_type.is_array:
3417 3418 3419
            # we need a ptr type here instead of an array type, as
            # array types can result in invalid type casts in the C
            # code
3420
            self.type = PyrexTypes.CPtrType(base_type.base_type)
3421 3422 3423
        else:
            self.base = self.base.coerce_to_pyobject(env)
            self.type = py_object_type
3424 3425 3426
        if base_type.is_builtin_type:
            # slicing builtin types returns something of the same type
            self.type = base_type
3427
        c_int = PyrexTypes.c_py_ssize_t_type
William Stein's avatar
William Stein committed
3428 3429 3430 3431 3432
        if self.start:
            self.start = self.start.coerce_to(c_int, env)
        if self.stop:
            self.stop = self.stop.coerce_to(c_int, env)
        self.is_temp = 1
3433

3434
    nogil_check = Node.gil_error
3435 3436
    gil_message = "Slicing Python object"

William Stein's avatar
William Stein committed
3437
    def generate_result_code(self, code):
3438 3439 3440 3441
        if not self.type.is_pyobject:
            error(self.pos,
                  "Slicing is not currently supported for '%s'." % self.type)
            return
Robert Bradshaw's avatar
Robert Bradshaw committed
3442
        if self.base.type.is_string:
3443 3444 3445
            base_result = self.base.result()
            if self.base.type != PyrexTypes.c_char_ptr_type:
                base_result = '((const char*)%s)' % base_result
Robert Bradshaw's avatar
Robert Bradshaw committed
3446 3447
            if self.stop is None:
                code.putln(
3448
                    "%s = PyBytes_FromString(%s + %s); %s" % (
Robert Bradshaw's avatar
Robert Bradshaw committed
3449
                        self.result(),
3450
                        base_result,
Robert Bradshaw's avatar
Robert Bradshaw committed
3451 3452 3453 3454
                        self.start_code(),
                        code.error_goto_if_null(self.result(), self.pos)))
            else:
                code.putln(
3455
                    "%s = PyBytes_FromStringAndSize(%s + %s, %s - %s); %s" % (
Robert Bradshaw's avatar
Robert Bradshaw committed
3456
                        self.result(),
3457
                        base_result,
Robert Bradshaw's avatar
Robert Bradshaw committed
3458 3459 3460 3461 3462 3463
                        self.start_code(),
                        self.stop_code(),
                        self.start_code(),
                        code.error_goto_if_null(self.result(), self.pos)))
        else:
            code.putln(
3464
                "%s = __Pyx_PySequence_GetSlice(%s, %s, %s); %s" % (
Robert Bradshaw's avatar
Robert Bradshaw committed
3465 3466 3467 3468 3469
                    self.result(),
                    self.base.py_result(),
                    self.start_code(),
                    self.stop_code(),
                    code.error_goto_if_null(self.result(), self.pos)))
3470
        code.put_gotref(self.py_result())
3471

William Stein's avatar
William Stein committed
3472 3473
    def generate_assignment_code(self, rhs, code):
        self.generate_subexpr_evaluation_code(code)
3474
        if self.type.is_pyobject:
3475
            code.put_error_if_neg(self.pos,
3476
                "__Pyx_PySequence_SetSlice(%s, %s, %s, %s)" % (
3477 3478 3479
                    self.base.py_result(),
                    self.start_code(),
                    self.stop_code(),
Lisandro Dalcin's avatar
Lisandro Dalcin committed
3480
                    rhs.py_result()))
3481 3482 3483 3484 3485 3486 3487 3488
        else:
            start_offset = ''
            if self.start:
                start_offset = self.start_code()
                if start_offset == '0':
                    start_offset = ''
                else:
                    start_offset += '+'
Stefan Behnel's avatar
Stefan Behnel committed
3489 3490
            if rhs.type.is_array:
                array_length = rhs.type.size
3491
                self.generate_slice_guard_code(code, array_length)
Stefan Behnel's avatar
Stefan Behnel committed
3492
            else:
Stefan Behnel's avatar
Stefan Behnel committed
3493 3494
                error(self.pos,
                      "Slice assignments from pointers are not yet supported.")
Stefan Behnel's avatar
Stefan Behnel committed
3495 3496
                # FIXME: fix the array size according to start/stop
                array_length = self.base.type.size
3497 3498 3499 3500
            for i in range(array_length):
                code.putln("%s[%s%s] = %s[%d];" % (
                        self.base.result(), start_offset, i,
                        rhs.result(), i))
William Stein's avatar
William Stein committed
3501
        self.generate_subexpr_disposal_code(code)
3502
        self.free_subexpr_temps(code)
William Stein's avatar
William Stein committed
3503
        rhs.generate_disposal_code(code)
3504
        rhs.free_temps(code)
William Stein's avatar
William Stein committed
3505 3506

    def generate_deletion_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
3507
        if not self.base.type.is_pyobject:
3508 3509 3510
            error(self.pos,
                  "Deleting slices is only supported for Python types, not '%s'." % self.type)
            return
William Stein's avatar
William Stein committed
3511
        self.generate_subexpr_evaluation_code(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
3512
        code.put_error_if_neg(self.pos,
3513
            "__Pyx_PySequence_DelSlice(%s, %s, %s)" % (
William Stein's avatar
William Stein committed
3514 3515
                self.base.py_result(),
                self.start_code(),
Robert Bradshaw's avatar
Robert Bradshaw committed
3516
                self.stop_code()))
William Stein's avatar
William Stein committed
3517
        self.generate_subexpr_disposal_code(code)
3518
        self.free_subexpr_temps(code)
3519 3520 3521 3522 3523 3524 3525 3526 3527 3528

    def generate_slice_guard_code(self, code, target_size):
        if not self.base.type.is_array:
            return
        slice_size = self.base.type.size
        start = stop = None
        if self.stop:
            stop = self.stop.result()
            try:
                stop = int(stop)
Stefan Behnel's avatar
Stefan Behnel committed
3529
                if stop < 0:
3530
                    slice_size = self.base.type.size + stop
Stefan Behnel's avatar
Stefan Behnel committed
3531 3532
                else:
                    slice_size = stop
3533 3534 3535 3536 3537 3538 3539 3540 3541 3542 3543 3544 3545 3546 3547 3548 3549 3550 3551 3552 3553 3554 3555 3556 3557 3558 3559 3560 3561 3562
                stop = None
            except ValueError:
                pass
        if self.start:
            start = self.start.result()
            try:
                start = int(start)
                if start < 0:
                    start = self.base.type.size + start
                slice_size -= start
                start = None
            except ValueError:
                pass
        check = None
        if slice_size < 0:
            if target_size > 0:
                error(self.pos, "Assignment to empty slice.")
        elif start is None and stop is None:
            # we know the exact slice length
            if target_size != slice_size:
                error(self.pos, "Assignment to slice of wrong length, expected %d, got %d" % (
                        slice_size, target_size))
        elif start is not None:
            if stop is None:
                stop = slice_size
            check = "(%s)-(%s)" % (stop, start)
        else: # stop is not None:
            check = stop
        if check:
            code.putln("if (unlikely((%s) != %d)) {" % (check, target_size))
cgohlke's avatar
cgohlke committed
3563
            code.putln('PyErr_Format(PyExc_ValueError, "Assignment to slice of wrong length, expected %%" CYTHON_FORMAT_SSIZE_T "d, got %%" CYTHON_FORMAT_SSIZE_T "d", (Py_ssize_t)%d, (Py_ssize_t)(%s));' % (
3564 3565 3566
                        target_size, check))
            code.putln(code.error_goto(self.pos))
            code.putln("}")
3567

William Stein's avatar
William Stein committed
3568 3569
    def start_code(self):
        if self.start:
3570
            return self.start.result()
William Stein's avatar
William Stein committed
3571 3572
        else:
            return "0"
3573

William Stein's avatar
William Stein committed
3574 3575
    def stop_code(self):
        if self.stop:
3576
            return self.stop.result()
3577 3578
        elif self.base.type.is_array:
            return self.base.type.size
William Stein's avatar
William Stein committed
3579
        else:
3580
            return "PY_SSIZE_T_MAX"
3581

William Stein's avatar
William Stein committed
3582
    def calculate_result_code(self):
3583
        # self.result() is not used, but this method must exist
William Stein's avatar
William Stein committed
3584
        return "<unused>"
3585

William Stein's avatar
William Stein committed
3586

3587
class SliceNode(ExprNode):
William Stein's avatar
William Stein committed
3588 3589 3590 3591 3592
    #  start:stop:step in subscript list
    #
    #  start     ExprNode
    #  stop      ExprNode
    #  step      ExprNode
3593

3594 3595
    subexprs = ['start', 'stop', 'step']

3596 3597
    type = py_object_type
    is_temp = 1
3598 3599

    def calculate_constant_result(self):
3600 3601 3602 3603
        self.constant_result = slice(
            self.start.constant_result,
            self.stop.constant_result,
            self.step.constant_result)
3604

3605 3606
    def compile_time_value(self, denv):
        start = self.start.compile_time_value(denv)
Stefan Behnel's avatar
Stefan Behnel committed
3607 3608
        stop = self.stop.compile_time_value(denv)
        step = self.step.compile_time_value(denv)
3609 3610 3611 3612 3613
        try:
            return slice(start, stop, step)
        except Exception, e:
            self.compile_time_value_error(e)

William Stein's avatar
William Stein committed
3614 3615 3616 3617 3618 3619 3620
    def analyse_types(self, env):
        self.start.analyse_types(env)
        self.stop.analyse_types(env)
        self.step.analyse_types(env)
        self.start = self.start.coerce_to_pyobject(env)
        self.stop = self.stop.coerce_to_pyobject(env)
        self.step = self.step.coerce_to_pyobject(env)
3621 3622 3623
        if self.start.is_literal and self.stop.is_literal and self.step.is_literal:
            self.is_literal = True
            self.is_temp = False
3624 3625 3626

    gil_message = "Constructing Python slice object"

3627 3628 3629
    def calculate_result_code(self):
        return self.result_code

William Stein's avatar
William Stein committed
3630
    def generate_result_code(self, code):
3631 3632 3633 3634 3635
        if self.is_literal:
            self.result_code = code.get_py_const(py_object_type, 'slice_', cleanup_level=2)
            code = code.get_cached_constants_writer()
            code.mark_pos(self.pos)

William Stein's avatar
William Stein committed
3636
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
3637
            "%s = PySlice_New(%s, %s, %s); %s" % (
3638
                self.result(),
3639 3640
                self.start.py_result(),
                self.stop.py_result(),
William Stein's avatar
William Stein committed
3641
                self.step.py_result(),
3642
                code.error_goto_if_null(self.result(), self.pos)))
3643
        code.put_gotref(self.py_result())
3644 3645
        if self.is_literal:
            code.put_giveref(self.py_result())
William Stein's avatar
William Stein committed
3646

3647 3648 3649 3650 3651 3652 3653 3654 3655 3656 3657 3658 3659
    def __deepcopy__(self, memo):
        """
        There is a copy bug in python 2.4 for slice objects.
        """
        return SliceNode(
            self.pos,
            start=copy.deepcopy(self.start, memo),
            stop=copy.deepcopy(self.stop, memo),
            step=copy.deepcopy(self.step, memo),
            is_temp=self.is_temp,
            is_literal=self.is_literal,
            constant_result=self.constant_result)

3660

3661
class CallNode(ExprNode):
3662

Stefan Behnel's avatar
Stefan Behnel committed
3663 3664 3665
    # allow overriding the default 'may_be_none' behaviour
    may_return_none = None

3666 3667 3668 3669 3670 3671 3672 3673 3674 3675 3676 3677 3678 3679 3680 3681 3682 3683 3684 3685 3686
    def infer_type(self, env):
        function = self.function
        func_type = function.infer_type(env)
        if isinstance(self.function, NewExprNode):
            return PyrexTypes.CPtrType(self.function.class_type)
        if func_type.is_ptr:
            func_type = func_type.base_type
        if func_type.is_cfunction:
            return func_type.return_type
        elif func_type is type_type:
            if function.is_name and function.entry and function.entry.type:
                result_type = function.entry.type
                if result_type.is_extension_type:
                    return result_type
                elif result_type.is_builtin_type:
                    if function.entry.name == 'float':
                        return PyrexTypes.c_double_type
                    elif function.entry.name in Builtin.types_that_construct_their_instance:
                        return result_type
        return py_object_type

Robert Bradshaw's avatar
Robert Bradshaw committed
3687 3688 3689 3690 3691
    def type_dependencies(self, env):
        # TODO: Update when Danilo's C++ code merged in to handle the
        # the case of function overloading.
        return self.function.type_dependencies(env)

Stefan Behnel's avatar
Stefan Behnel committed
3692 3693 3694 3695 3696
    def may_be_none(self):
        if self.may_return_none is not None:
            return self.may_return_none
        return ExprNode.may_be_none(self)

Robert Bradshaw's avatar
Robert Bradshaw committed
3697 3698 3699 3700 3701 3702
    def analyse_as_type_constructor(self, env):
        type = self.function.analyse_as_type(env)
        if type and type.is_struct_or_union:
            args, kwds = self.explicit_args_kwds()
            items = []
            for arg, member in zip(args, type.scope.var_entries):
3703
                items.append(DictItemNode(pos=arg.pos, key=StringNode(pos=arg.pos, value=member.name), value=arg))
Robert Bradshaw's avatar
Robert Bradshaw committed
3704 3705 3706 3707 3708 3709 3710
            if kwds:
                items += kwds.key_value_pairs
            self.key_value_pairs = items
            self.__class__ = DictNode
            self.analyse_types(env)
            self.coerce_to(type, env)
            return True
3711 3712 3713 3714 3715 3716 3717 3718
        elif type and type.is_cpp_class:
            for arg in self.args:
                arg.analyse_types(env)
            constructor = type.scope.lookup("<init>")
            self.function = RawCNameExprNode(self.function.pos, constructor.type)
            self.function.entry = constructor
            self.function.set_cname(type.declaration_code(""))
            self.analyse_c_function_call(env)
3719
            self.type = type
3720
            return True
3721

3722 3723
    def is_lvalue(self):
        return self.type.is_reference
3724

3725
    def nogil_check(self, env):
3726 3727
        func_type = self.function_type()
        if func_type.is_pyobject:
3728
            self.gil_error()
3729
        elif not getattr(func_type, 'nogil', False):
3730
            self.gil_error()
3731 3732 3733

    gil_message = "Calling gil-requiring function"

3734 3735

class SimpleCallNode(CallNode):
William Stein's avatar
William Stein committed
3736 3737 3738 3739 3740 3741 3742
    #  Function call without keyword, * or ** args.
    #
    #  function       ExprNode
    #  args           [ExprNode]
    #  arg_tuple      ExprNode or None     used internally
    #  self           ExprNode or None     used internally
    #  coerced_self   ExprNode or None     used internally
3743
    #  wrapper_call   bool                 used internally
3744
    #  has_optional_args   bool            used internally
3745
    #  nogil          bool                 used internally
3746

William Stein's avatar
William Stein committed
3747
    subexprs = ['self', 'coerced_self', 'function', 'args', 'arg_tuple']
3748

William Stein's avatar
William Stein committed
3749 3750 3751
    self = None
    coerced_self = None
    arg_tuple = None
3752
    wrapper_call = False
3753
    has_optional_args = False
3754
    nogil = False
3755
    analysed = False
3756

3757 3758 3759 3760 3761 3762 3763
    def compile_time_value(self, denv):
        function = self.function.compile_time_value(denv)
        args = [arg.compile_time_value(denv) for arg in self.args]
        try:
            return function(*args)
        except Exception, e:
            self.compile_time_value_error(e)
3764

3765
    def analyse_as_type(self, env):
3766
        attr = self.function.as_cython_attribute()
3767 3768 3769 3770 3771 3772 3773 3774 3775 3776 3777 3778
        if attr == 'pointer':
            if len(self.args) != 1:
                error(self.args.pos, "only one type allowed.")
            else:
                type = self.args[0].analyse_as_type(env)
                if not type:
                    error(self.args[0].pos, "Unknown type")
                else:
                    return PyrexTypes.CPtrType(type)

    def explicit_args_kwds(self):
        return self.args, None
3779

William Stein's avatar
William Stein committed
3780
    def analyse_types(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
3781 3782
        if self.analyse_as_type_constructor(env):
            return
3783 3784 3785
        if self.analysed:
            return
        self.analysed = True
William Stein's avatar
William Stein committed
3786 3787 3788
        function = self.function
        function.is_called = 1
        self.function.analyse_types(env)
Mark Florisson's avatar
Mark Florisson committed
3789

William Stein's avatar
William Stein committed
3790 3791 3792 3793 3794
        if function.is_attribute and function.entry and function.entry.is_cmethod:
            # Take ownership of the object from which the attribute
            # was obtained, because we need to pass it as 'self'.
            self.self = function.obj
            function.obj = CloneNode(self.self)
Mark Florisson's avatar
Mark Florisson committed
3795

William Stein's avatar
William Stein committed
3796 3797
        func_type = self.function_type()
        if func_type.is_pyobject:
3798 3799
            self.arg_tuple = TupleNode(self.pos, args = self.args)
            self.arg_tuple.analyse_types(env)
William Stein's avatar
William Stein committed
3800
            self.args = None
3801 3802 3803
            if func_type is Builtin.type_type and function.is_name and \
                   function.entry and \
                   function.entry.is_builtin and \
3804 3805 3806 3807 3808 3809 3810 3811 3812
                   function.entry.name in Builtin.types_that_construct_their_instance:
                # calling a builtin type that returns a specific object type
                if function.entry.name == 'float':
                    # the following will come true later on in a transform
                    self.type = PyrexTypes.c_double_type
                    self.result_ctype = PyrexTypes.c_double_type
                else:
                    self.type = Builtin.builtin_types[function.entry.name]
                    self.result_ctype = py_object_type
Stefan Behnel's avatar
Stefan Behnel committed
3813
                self.may_return_none = False
3814
            elif function.is_name and function.type_entry:
3815 3816 3817 3818 3819
                # We are calling an extension type constructor.  As
                # long as we do not support __new__(), the result type
                # is clear
                self.type = function.type_entry.type
                self.result_ctype = py_object_type
Stefan Behnel's avatar
Stefan Behnel committed
3820
                self.may_return_none = False
3821 3822
            else:
                self.type = py_object_type
William Stein's avatar
William Stein committed
3823 3824 3825 3826
            self.is_temp = 1
        else:
            for arg in self.args:
                arg.analyse_types(env)
3827

William Stein's avatar
William Stein committed
3828 3829
            if self.self and func_type.args:
                # Coerce 'self' to the type expected by the method.
3830 3831 3832
                self_arg = func_type.args[0]
                if self_arg.not_none: # C methods must do the None test for self at *call* time
                    self.self = self.self.as_none_safe_node(
3833 3834 3835
                        "'NoneType' object has no attribute '%s'",
                        error = 'PyExc_AttributeError',
                        format_args = [self.function.entry.name])
3836
                expected_type = self_arg.type
Stefan Behnel's avatar
Stefan Behnel committed
3837 3838 3839 3840 3841
                if self_arg.accept_builtin_subtypes:
                    self.coerced_self = CMethodSelfCloneNode(self.self)
                else:
                    self.coerced_self = CloneNode(self.self)
                self.coerced_self = self.coerced_self.coerce_to(expected_type, env)
William Stein's avatar
William Stein committed
3842 3843 3844
                # Insert coerced 'self' argument into argument list.
                self.args.insert(0, self.coerced_self)
            self.analyse_c_function_call(env)
3845

William Stein's avatar
William Stein committed
3846 3847
    def function_type(self):
        # Return the type of the function being called, coercing a function
3848 3849
        # pointer to a function if necessary. If the function has fused
        # arguments, return the specific type.
William Stein's avatar
William Stein committed
3850
        func_type = self.function.type
3851

William Stein's avatar
William Stein committed
3852 3853
        if func_type.is_ptr:
            func_type = func_type.base_type
3854

William Stein's avatar
William Stein committed
3855
        return func_type
3856

3857 3858 3859 3860 3861 3862 3863
    def is_simple(self):
        # C function calls could be considered simple, but they may
        # have side-effects that may hit when multiple operations must
        # be effected in order, e.g. when constructing the argument
        # sequence for a function call or comparing values.
        return False

William Stein's avatar
William Stein committed
3864
    def analyse_c_function_call(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
3865
        if self.function.type is error_type:
3866
            self.type = error_type
Robert Bradshaw's avatar
Robert Bradshaw committed
3867
            return
3868

Robert Bradshaw's avatar
Robert Bradshaw committed
3869
        if self.function.type.is_cpp_class:
3870 3871
            overloaded_entry = self.function.type.scope.lookup("operator()")
            if overloaded_entry is None:
Robert Bradshaw's avatar
Robert Bradshaw committed
3872 3873 3874
                self.type = PyrexTypes.error_type
                self.result_code = "<error>"
                return
3875 3876
        elif hasattr(self.function, 'entry'):
            overloaded_entry = self.function.entry
3877
        elif (isinstance(self.function, IndexNode) and
3878
              self.function.is_fused_index):
3879
            overloaded_entry = self.function.type.entry
Robert Bradshaw's avatar
Robert Bradshaw committed
3880
        else:
3881
            overloaded_entry = None
3882

3883
        if overloaded_entry:
3884
            if self.function.type.is_fused:
3885
                functypes = self.function.type.get_all_specialized_function_types()
Mark Florisson's avatar
Mark Florisson committed
3886
                alternatives = [f.entry for f in functypes]
3887 3888 3889 3890 3891
            else:
                alternatives = overloaded_entry.all_alternatives()

            entry = PyrexTypes.best_match(self.args, alternatives, self.pos, env)

3892 3893 3894 3895
            if not entry:
                self.type = PyrexTypes.error_type
                self.result_code = "<error>"
                return
Mark Florisson's avatar
Mark Florisson committed
3896 3897

            entry.used = True
3898 3899
            self.function.entry = entry
            self.function.type = entry.type
3900 3901 3902 3903 3904 3905 3906 3907
            func_type = self.function_type()
        else:
            func_type = self.function_type()
            if not func_type.is_cfunction:
                error(self.pos, "Calling non-function type '%s'" % func_type)
                self.type = PyrexTypes.error_type
                self.result_code = "<error>"
                return
William Stein's avatar
William Stein committed
3908
        # Check no. of args
3909 3910
        max_nargs = len(func_type.args)
        expected_nargs = max_nargs - func_type.optional_arg_count
William Stein's avatar
William Stein committed
3911
        actual_nargs = len(self.args)
3912 3913 3914
        if func_type.optional_arg_count and expected_nargs != actual_nargs:
            self.has_optional_args = 1
            self.is_temp = 1
William Stein's avatar
William Stein committed
3915
        # Coerce arguments
3916
        some_args_in_temps = False
3917
        for i in xrange(min(max_nargs, actual_nargs)):
William Stein's avatar
William Stein committed
3918
            formal_type = func_type.args[i].type
3919
            arg = self.args[i].coerce_to(formal_type, env)
3920
            if arg.is_temp:
3921 3922
                if i > 0:
                    # first argument in temp doesn't impact subsequent arguments
3923
                    some_args_in_temps = True
3924
            elif arg.type.is_pyobject and not env.nogil:
3925 3926
                if i == 0 and self.self is not None:
                    # a method's cloned "self" argument is ok
3927
                    pass
3928
                elif arg.nonlocally_immutable():
3929 3930 3931
                    # plain local variables are ok
                    pass
                else:
3932 3933 3934 3935
                    # we do not safely own the argument's reference,
                    # but we must make sure it cannot be collected
                    # before we return from the function, so we create
                    # an owned temp reference to it
3936 3937
                    if i > 0: # first argument doesn't matter
                        some_args_in_temps = True
3938
                    arg = arg.coerce_to_temp(env)
3939
            self.args[i] = arg
3940
        # handle additional varargs parameters
3941
        for i in xrange(max_nargs, actual_nargs):
3942 3943 3944 3945 3946 3947 3948
            arg = self.args[i]
            if arg.type.is_pyobject:
                arg_ctype = arg.type.default_coerced_ctype()
                if arg_ctype is None:
                    error(self.args[i].pos,
                          "Python object cannot be passed as a varargs parameter")
                else:
3949
                    self.args[i] = arg = arg.coerce_to(arg_ctype, env)
3950 3951
            if arg.is_temp and i > 0:
                some_args_in_temps = True
3952 3953 3954
        if some_args_in_temps:
            # if some args are temps and others are not, they may get
            # constructed in the wrong order (temps first) => make
3955 3956 3957 3958
            # sure they are either all temps or all not temps (except
            # for the last argument, which is evaluated last in any
            # case)
            for i in xrange(actual_nargs-1):
3959 3960
                if i == 0 and self.self is not None:
                    continue # self is ok
3961
                arg = self.args[i]
3962 3963
                if arg.nonlocally_immutable():
                    # locals, C functions, unassignable types are safe.
3964
                    pass
3965 3966
                elif arg.type.is_cpp_class:
                    # Assignment has side effects, avoid.
3967 3968
                    pass
                elif env.nogil and arg.type.is_pyobject:
3969 3970 3971
                    # can't copy a Python reference into a temp in nogil
                    # env (this is safe: a construction would fail in
                    # nogil anyway)
3972 3973
                    pass
                else:
3974 3975 3976 3977 3978
                    #self.args[i] = arg.coerce_to_temp(env)
                    # instead: issue a warning
                    if i > 0 or i == 1 and self.self is not None: # skip first arg
                        warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0)
                        break
3979

William Stein's avatar
William Stein committed
3980
        # Calc result type and code fragment
Robert Bradshaw's avatar
Robert Bradshaw committed
3981
        if isinstance(self.function, NewExprNode):
3982
            self.type = PyrexTypes.CPtrType(self.function.class_type)
Robert Bradshaw's avatar
Robert Bradshaw committed
3983 3984
        else:
            self.type = func_type.return_type
3985

3986 3987 3988
        if self.function.is_name or self.function.is_attribute:
            if self.function.entry and self.function.entry.utility_code:
                self.is_temp = 1 # currently doesn't work for self.calculate_result_code()
3989

Stefan Behnel's avatar
Stefan Behnel committed
3990 3991 3992 3993 3994 3995
        if self.type.is_pyobject:
            self.result_ctype = py_object_type
            self.is_temp = 1
        elif func_type.exception_value is not None \
                 or func_type.exception_check:
            self.is_temp = 1
3996 3997 3998 3999
        elif self.type.is_memoryviewslice:
            self.is_temp = 1
            # func_type.exception_check = True

4000
        # Called in 'nogil' context?
4001
        self.nogil = env.nogil
4002 4003 4004 4005 4006
        if (self.nogil and
            func_type.exception_check and
            func_type.exception_check != '+'):
            env.use_utility_code(pyerr_occurred_withgil_utility_code)
        # C++ exception handler
Robert Bradshaw's avatar
Robert Bradshaw committed
4007 4008
        if func_type.exception_check == '+':
            if func_type.exception_value is None:
4009
                env.use_utility_code(UtilityCode.load_cached("CppExceptionConversion", "CppSupport.cpp"))
Robert Bradshaw's avatar
Robert Bradshaw committed
4010

William Stein's avatar
William Stein committed
4011 4012
    def calculate_result_code(self):
        return self.c_call_code()
4013

William Stein's avatar
William Stein committed
4014 4015
    def c_call_code(self):
        func_type = self.function_type()
4016
        if self.type is PyrexTypes.error_type or not func_type.is_cfunction:
William Stein's avatar
William Stein committed
4017 4018 4019
            return "<error>"
        formal_args = func_type.args
        arg_list_code = []
4020
        args = list(zip(formal_args, self.args))
4021 4022 4023 4024
        max_nargs = len(func_type.args)
        expected_nargs = max_nargs - func_type.optional_arg_count
        actual_nargs = len(self.args)
        for formal_arg, actual_arg in args[:expected_nargs]:
William Stein's avatar
William Stein committed
4025 4026
                arg_code = actual_arg.result_as(formal_arg.type)
                arg_list_code.append(arg_code)
4027

4028 4029
        if func_type.is_overridable:
            arg_list_code.append(str(int(self.wrapper_call or self.function.entry.is_unbound_cmethod)))
4030

4031
        if func_type.optional_arg_count:
4032
            if expected_nargs == actual_nargs:
4033
                optional_args = 'NULL'
4034
            else:
4035
                optional_args = "&%s" % self.opt_arg_struct
4036
            arg_list_code.append(optional_args)
4037

William Stein's avatar
William Stein committed
4038
        for actual_arg in self.args[len(formal_args):]:
4039
            arg_list_code.append(actual_arg.result())
4040 4041

        result = "%s(%s)" % (self.function.result(), ', '.join(arg_list_code))
William Stein's avatar
William Stein committed
4042
        return result
4043

William Stein's avatar
William Stein committed
4044 4045
    def generate_result_code(self, code):
        func_type = self.function_type()
4046 4047 4048
        if self.function.is_name or self.function.is_attribute:
            if self.function.entry and self.function.entry.utility_code:
                code.globalstate.use_utility_code(self.function.entry.utility_code)
William Stein's avatar
William Stein committed
4049
        if func_type.is_pyobject:
4050
            arg_code = self.arg_tuple.py_result()
William Stein's avatar
William Stein committed
4051
            code.putln(
4052
                "%s = PyObject_Call(%s, %s, NULL); %s" % (
4053
                    self.result(),
William Stein's avatar
William Stein committed
4054
                    self.function.py_result(),
4055
                    arg_code,
4056
                    code.error_goto_if_null(self.result(), self.pos)))
4057
            code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
4058
        elif func_type.is_cfunction:
4059 4060 4061
            if self.has_optional_args:
                actual_nargs = len(self.args)
                expected_nargs = len(func_type.args) - func_type.optional_arg_count
4062 4063
                self.opt_arg_struct = code.funcstate.allocate_temp(
                    func_type.op_arg_struct.base_type, manage_ref=True)
4064 4065 4066 4067
                code.putln("%s.%s = %s;" % (
                        self.opt_arg_struct,
                        Naming.pyrex_prefix + "n",
                        len(self.args) - expected_nargs))
4068
                args = list(zip(func_type.args, self.args))
4069 4070 4071
                for formal_arg, actual_arg in args[expected_nargs:actual_nargs]:
                    code.putln("%s.%s = %s;" % (
                            self.opt_arg_struct,
4072
                            func_type.opt_arg_cname(formal_arg.name),
4073
                            actual_arg.result_as(formal_arg.type)))
William Stein's avatar
William Stein committed
4074
            exc_checks = []
4075
            if self.type.is_pyobject and self.is_temp:
4076
                exc_checks.append("!%s" % self.result())
4077 4078 4079
            elif self.type.is_memoryviewslice:
                assert self.is_temp
                exc_checks.append(self.type.error_condition(self.result()))
William Stein's avatar
William Stein committed
4080
            else:
4081 4082
                exc_val = func_type.exception_value
                exc_check = func_type.exception_check
William Stein's avatar
William Stein committed
4083
                if exc_val is not None:
4084
                    exc_checks.append("%s == %s" % (self.result(), exc_val))
William Stein's avatar
William Stein committed
4085
                if exc_check:
4086 4087
                    if self.nogil:
                        exc_checks.append("__Pyx_ErrOccurredWithGIL()")
4088
                    else:
4089
                        exc_checks.append("PyErr_Occurred()")
William Stein's avatar
William Stein committed
4090 4091
            if self.is_temp or exc_checks:
                rhs = self.c_call_code()
4092 4093
                if self.result():
                    lhs = "%s = " % self.result()
William Stein's avatar
William Stein committed
4094 4095 4096
                    if self.is_temp and self.type.is_pyobject:
                        #return_type = self.type # func_type.return_type
                        #print "SimpleCallNode.generate_result_code: casting", rhs, \
Robert Bradshaw's avatar
Robert Bradshaw committed
4097
                        #    "from", return_type, "to pyobject" ###
William Stein's avatar
William Stein committed
4098 4099 4100
                        rhs = typecast(py_object_type, self.type, rhs)
                else:
                    lhs = ""
Felix Wu's avatar
Felix Wu committed
4101
                if func_type.exception_check == '+':
Robert Bradshaw's avatar
Robert Bradshaw committed
4102 4103 4104
                    if func_type.exception_value is None:
                        raise_py_exception = "__Pyx_CppExn2PyErr()"
                    elif func_type.exception_value.type.is_pyobject:
4105 4106 4107
                        raise_py_exception = ' try { throw; } catch(const std::exception& exn) { PyErr_SetString(%s, exn.what()); } catch(...) { PyErr_SetNone(%s); }' % (
                            func_type.exception_value.entry.cname,
                            func_type.exception_value.entry.cname)
Robert Bradshaw's avatar
Robert Bradshaw committed
4108 4109
                    else:
                        raise_py_exception = '%s(); if (!PyErr_Occurred()) PyErr_SetString(PyExc_RuntimeError , "Error converting c++ exception.")' % func_type.exception_value.entry.cname
4110 4111
                    if self.nogil:
                        raise_py_exception = 'Py_BLOCK_THREADS; %s; Py_UNBLOCK_THREADS' % raise_py_exception
Felix Wu's avatar
Felix Wu committed
4112
                    code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
4113
                    "try {%s%s;} catch(...) {%s; %s}" % (
Felix Wu's avatar
Felix Wu committed
4114 4115
                        lhs,
                        rhs,
Robert Bradshaw's avatar
Robert Bradshaw committed
4116
                        raise_py_exception,
Felix Wu's avatar
Felix Wu committed
4117
                        code.error_goto(self.pos)))
4118 4119 4120 4121 4122 4123
                else:
                    if exc_checks:
                        goto_error = code.error_goto_if(" && ".join(exc_checks), self.pos)
                    else:
                        goto_error = ""
                    code.putln("%s%s; %s" % (lhs, rhs, goto_error))
4124
                if self.type.is_pyobject and self.result():
4125
                    code.put_gotref(self.py_result())
4126 4127
            if self.has_optional_args:
                code.funcstate.release_temp(self.opt_arg_struct)
4128 4129


4130 4131 4132 4133 4134 4135 4136 4137 4138 4139 4140 4141 4142 4143 4144 4145 4146 4147 4148 4149 4150 4151 4152 4153 4154 4155 4156 4157 4158 4159 4160 4161 4162 4163 4164 4165 4166 4167 4168 4169 4170 4171 4172 4173 4174 4175 4176 4177 4178 4179 4180 4181 4182 4183 4184 4185 4186 4187 4188 4189 4190 4191 4192 4193 4194 4195 4196 4197 4198 4199 4200 4201 4202 4203 4204 4205 4206 4207 4208 4209 4210 4211 4212 4213
class InlinedDefNodeCallNode(CallNode):
    #  Inline call to defnode
    #
    #  function       PyCFunctionNode
    #  function_name  NameNode
    #  args           [ExprNode]

    subexprs = ['args', 'function_name']
    is_temp = 1
    type = py_object_type
    function = None
    function_name = None

    def can_be_inlined(self):
        func_type= self.function.def_node
        if func_type.star_arg or func_type.starstar_arg:
            return False
        if len(func_type.args) != len(self.args):
            return False
        return True

    def analyse_types(self, env):
        self.function_name.analyse_types(env)

        for arg in self.args:
            arg.analyse_types(env)

        func_type = self.function.def_node
        actual_nargs = len(self.args)

        # Coerce arguments
        some_args_in_temps = False
        for i in xrange(actual_nargs):
            formal_type = func_type.args[i].type
            arg = self.args[i].coerce_to(formal_type, env)
            if arg.is_temp:
                if i > 0:
                    # first argument in temp doesn't impact subsequent arguments
                    some_args_in_temps = True
            elif arg.type.is_pyobject and not env.nogil:
                if arg.nonlocally_immutable():
                    # plain local variables are ok
                    pass
                else:
                    # we do not safely own the argument's reference,
                    # but we must make sure it cannot be collected
                    # before we return from the function, so we create
                    # an owned temp reference to it
                    if i > 0: # first argument doesn't matter
                        some_args_in_temps = True
                    arg = arg.coerce_to_temp(env)
            self.args[i] = arg

        if some_args_in_temps:
            # if some args are temps and others are not, they may get
            # constructed in the wrong order (temps first) => make
            # sure they are either all temps or all not temps (except
            # for the last argument, which is evaluated last in any
            # case)
            for i in xrange(actual_nargs-1):
                arg = self.args[i]
                if arg.nonlocally_immutable():
                    # locals, C functions, unassignable types are safe.
                    pass
                elif arg.type.is_cpp_class:
                    # Assignment has side effects, avoid.
                    pass
                elif env.nogil and arg.type.is_pyobject:
                    # can't copy a Python reference into a temp in nogil
                    # env (this is safe: a construction would fail in
                    # nogil anyway)
                    pass
                else:
                    #self.args[i] = arg.coerce_to_temp(env)
                    # instead: issue a warning
                    if i > 0:
                        warning(arg.pos, "Argument evaluation order in C function call is undefined and may not be as expected", 0)
                        break

    def generate_result_code(self, code):
        arg_code = [self.function_name.py_result()]
        func_type = self.function.def_node
        for arg, proto_arg in zip(self.args, func_type.args):
            if arg.type.is_pyobject:
4214
                arg_code.append(arg.result_as(proto_arg.type))
4215 4216 4217 4218 4219 4220 4221 4222 4223 4224 4225 4226
            else:
                arg_code.append(arg.result())
        arg_code = ', '.join(arg_code)
        code.putln(
            "%s = %s(%s); %s" % (
                self.result(),
                self.function.def_node.entry.pyfunc_cname,
                arg_code,
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())


4227 4228
class PythonCapiFunctionNode(ExprNode):
    subexprs = []
4229
    def __init__(self, pos, py_name, cname, func_type, utility_code = None):
4230
        self.pos = pos
4231 4232
        self.name = py_name
        self.cname = cname
4233 4234 4235
        self.type = func_type
        self.utility_code = utility_code

4236 4237 4238
    def analyse_types(self, env):
        pass

4239 4240 4241 4242 4243
    def generate_result_code(self, code):
        if self.utility_code:
            code.globalstate.use_utility_code(self.utility_code)

    def calculate_result_code(self):
4244
        return self.cname
4245 4246 4247 4248

class PythonCapiCallNode(SimpleCallNode):
    # Python C-API Function call (only created in transforms)

Stefan Behnel's avatar
Stefan Behnel committed
4249 4250 4251 4252 4253 4254
    # By default, we assume that the call never returns None, as this
    # is true for most C-API functions in CPython.  If this does not
    # apply to a call, set the following to True (or None to inherit
    # the default behaviour).
    may_return_none = False

4255
    def __init__(self, pos, function_name, func_type,
4256
                 utility_code = None, py_name=None, **kwargs):
4257 4258 4259
        self.type = func_type.return_type
        self.result_ctype = self.type
        self.function = PythonCapiFunctionNode(
4260
            pos, py_name, function_name, func_type,
4261 4262 4263 4264 4265
            utility_code = utility_code)
        # call this last so that we can override the constructed
        # attributes above with explicit keyword arguments if required
        SimpleCallNode.__init__(self, pos, **kwargs)

William Stein's avatar
William Stein committed
4266

4267
class GeneralCallNode(CallNode):
William Stein's avatar
William Stein committed
4268 4269 4270 4271 4272 4273
    #  General Python function call, including keyword,
    #  * and ** arguments.
    #
    #  function         ExprNode
    #  positional_args  ExprNode          Tuple of positional arguments
    #  keyword_args     ExprNode or None  Dict of keyword arguments
4274

4275
    type = py_object_type
4276

4277
    subexprs = ['function', 'positional_args', 'keyword_args']
William Stein's avatar
William Stein committed
4278

4279
    nogil_check = Node.gil_error
4280

4281 4282 4283 4284 4285 4286 4287 4288
    def compile_time_value(self, denv):
        function = self.function.compile_time_value(denv)
        positional_args = self.positional_args.compile_time_value(denv)
        keyword_args = self.keyword_args.compile_time_value(denv)
        try:
            return function(*positional_args, **keyword_args)
        except Exception, e:
            self.compile_time_value_error(e)
4289

4290
    def explicit_args_kwds(self):
4291 4292
        if (self.keyword_args and not isinstance(self.keyword_args, DictNode) or
            not isinstance(self.positional_args, TupleNode)):
4293
            raise CompileError(self.pos,
4294 4295
                'Compile-time keyword arguments must be explicit.')
        return self.positional_args.args, self.keyword_args
4296

William Stein's avatar
William Stein committed
4297
    def analyse_types(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
4298 4299
        if self.analyse_as_type_constructor(env):
            return
William Stein's avatar
William Stein committed
4300 4301 4302 4303
        self.function.analyse_types(env)
        self.positional_args.analyse_types(env)
        if self.keyword_args:
            self.keyword_args.analyse_types(env)
4304
        if not self.function.type.is_pyobject:
4305 4306
            if self.function.type.is_error:
                self.type = error_type
Stefan Behnel's avatar
Stefan Behnel committed
4307
                return
4308
            if hasattr(self.function, 'entry') and not self.function.entry.as_variable:
4309
                error(self.pos, "Keyword and starred arguments not allowed in cdef functions.")
4310 4311
            else:
                self.function = self.function.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
4312 4313
        self.positional_args = \
            self.positional_args.coerce_to_pyobject(env)
Stefan Behnel's avatar
Stefan Behnel committed
4314
        function = self.function
4315 4316 4317 4318 4319
        if function.is_name and function.type_entry:
            # We are calling an extension type constructor.  As long
            # as we do not support __new__(), the result type is clear
            self.type = function.type_entry.type
            self.result_ctype = py_object_type
Stefan Behnel's avatar
Stefan Behnel committed
4320
            self.may_return_none = False
4321 4322
        else:
            self.type = py_object_type
William Stein's avatar
William Stein committed
4323
        self.is_temp = 1
4324

William Stein's avatar
William Stein committed
4325
    def generate_result_code(self, code):
4326
        if self.type.is_error: return
4327 4328
        if self.keyword_args:
            kwargs = self.keyword_args.py_result()
William Stein's avatar
William Stein committed
4329
        else:
4330
            kwargs = 'NULL'
William Stein's avatar
William Stein committed
4331
        code.putln(
4332
            "%s = PyObject_Call(%s, %s, %s); %s" % (
4333
                self.result(),
4334 4335 4336
                self.function.py_result(),
                self.positional_args.py_result(),
                kwargs,
4337
                code.error_goto_if_null(self.result(), self.pos)))
4338
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
4339 4340


4341
class AsTupleNode(ExprNode):
William Stein's avatar
William Stein committed
4342 4343 4344 4345
    #  Convert argument to tuple. Used for normalising
    #  the * argument of a function call.
    #
    #  arg    ExprNode
4346

William Stein's avatar
William Stein committed
4347
    subexprs = ['arg']
4348 4349 4350

    def calculate_constant_result(self):
        self.constant_result = tuple(self.base.constant_result)
4351

4352 4353 4354 4355 4356 4357 4358
    def compile_time_value(self, denv):
        arg = self.arg.compile_time_value(denv)
        try:
            return tuple(arg)
        except Exception, e:
            self.compile_time_value_error(e)

William Stein's avatar
William Stein committed
4359 4360 4361
    def analyse_types(self, env):
        self.arg.analyse_types(env)
        self.arg = self.arg.coerce_to_pyobject(env)
4362
        self.type = tuple_type
William Stein's avatar
William Stein committed
4363
        self.is_temp = 1
4364

4365 4366 4367
    def may_be_none(self):
        return False

4368
    nogil_check = Node.gil_error
4369 4370
    gil_message = "Constructing Python tuple"

William Stein's avatar
William Stein committed
4371 4372
    def generate_result_code(self, code):
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
4373
            "%s = PySequence_Tuple(%s); %s" % (
4374
                self.result(),
William Stein's avatar
William Stein committed
4375
                self.arg.py_result(),
4376
                code.error_goto_if_null(self.result(), self.pos)))
4377
        code.put_gotref(self.py_result())
4378

William Stein's avatar
William Stein committed
4379

4380
class AttributeNode(ExprNode):
William Stein's avatar
William Stein committed
4381 4382 4383 4384
    #  obj.attribute
    #
    #  obj          ExprNode
    #  attribute    string
4385
    #  needs_none_check boolean        Used if obj is an extension type.
4386
    #                                  If set to True, it is known that the type is not None.
William Stein's avatar
William Stein committed
4387 4388 4389 4390 4391 4392 4393
    #
    #  Used internally:
    #
    #  is_py_attr           boolean   Is a Python getattr operation
    #  member               string    C name of struct member
    #  is_called            boolean   Function call is being done on result
    #  entry                Entry     Symbol table entry of attribute
4394

William Stein's avatar
William Stein committed
4395 4396
    is_attribute = 1
    subexprs = ['obj']
4397

William Stein's avatar
William Stein committed
4398 4399 4400
    type = PyrexTypes.error_type
    entry = None
    is_called = 0
4401
    needs_none_check = True
4402
    is_memslice_transpose = False
William Stein's avatar
William Stein committed
4403

4404
    def as_cython_attribute(self):
Mark Florisson's avatar
Mark Florisson committed
4405 4406 4407
        if (isinstance(self.obj, NameNode) and
                self.obj.is_cython_module and not
                self.attribute == u"parallel"):
4408
            return self.attribute
Mark Florisson's avatar
Mark Florisson committed
4409

4410 4411 4412
        cy = self.obj.as_cython_attribute()
        if cy:
            return "%s.%s" % (cy, self.attribute)
4413
        return None
4414

4415 4416 4417 4418 4419 4420 4421 4422 4423
    def coerce_to(self, dst_type, env):
        #  If coercing to a generic pyobject and this is a cpdef function
        #  we can create the corresponding attribute
        if dst_type is py_object_type:
            entry = self.entry
            if entry and entry.is_cfunction and entry.as_variable:
                # must be a cpdef function
                self.is_temp = 1
                self.entry = entry.as_variable
4424
                self.analyse_as_python_attribute(env)
4425
                return self
4426
        return ExprNode.coerce_to(self, dst_type, env)
4427 4428 4429

    def calculate_constant_result(self):
        attr = self.attribute
4430
        if attr.startswith("__") and attr.endswith("__"):
4431 4432 4433
            return
        self.constant_result = getattr(self.obj.constant_result, attr)

4434 4435
    def compile_time_value(self, denv):
        attr = self.attribute
4436
        if attr.startswith("__") and attr.endswith("__"):
Stefan Behnel's avatar
Stefan Behnel committed
4437 4438
            error(self.pos,
                  "Invalid attribute name '%s' in compile-time expression" % attr)
4439
            return None
4440
        obj = self.obj.compile_time_value(denv)
4441 4442 4443 4444
        try:
            return getattr(obj, attr)
        except Exception, e:
            self.compile_time_value_error(e)
4445

Robert Bradshaw's avatar
Robert Bradshaw committed
4446 4447
    def type_dependencies(self, env):
        return self.obj.type_dependencies(env)
4448

4449 4450 4451 4452 4453 4454
    def infer_type(self, env):
        if self.analyse_as_cimported_attribute(env, 0):
            return self.entry.type
        elif self.analyse_as_unbound_cmethod(env):
            return self.entry.type
        else:
4455 4456 4457 4458 4459 4460 4461
            obj_type = self.obj.infer_type(env)
            self.analyse_attribute(env, obj_type = obj_type)
            if obj_type.is_builtin_type and self.type.is_cfunction:
                # special case: C-API replacements for C methods of
                # builtin types cannot be inferred as C functions as
                # that would prevent their use as bound methods
                self.type = py_object_type
4462
            return self.type
4463

William Stein's avatar
William Stein committed
4464 4465
    def analyse_target_declaration(self, env):
        pass
4466

William Stein's avatar
William Stein committed
4467 4468
    def analyse_target_types(self, env):
        self.analyse_types(env, target = 1)
Robert Bradshaw's avatar
Robert Bradshaw committed
4469 4470
        if self.type.is_const:
            error(self.pos, "Assignment to const attribute '%s'" % self.attribute)
4471 4472
        if not self.is_lvalue():
            error(self.pos, "Assignment to non-lvalue of type '%s'" % self.type)
4473

William Stein's avatar
William Stein committed
4474
    def analyse_types(self, env, target = 0):
4475
        self.initialized_check = env.directives['initializedcheck']
William Stein's avatar
William Stein committed
4476
        if self.analyse_as_cimported_attribute(env, target):
4477 4478 4479 4480 4481 4482 4483
            self.entry.used = True
        elif not target and self.analyse_as_unbound_cmethod(env):
            self.entry.used = True
        else:
            self.analyse_as_ordinary_attribute(env, target)
            if self.entry:
                self.entry.used = True
4484

4485 4486 4487 4488
        # may be mutated in a namenode now :)
        if self.is_attribute:
            self.wrap_obj_in_nonecheck(env)

William Stein's avatar
William Stein committed
4489 4490 4491 4492 4493 4494 4495 4496 4497 4498 4499 4500
    def analyse_as_cimported_attribute(self, env, target):
        # Try to interpret this as a reference to an imported
        # C const, type, var or function. If successful, mutates
        # this node into a NameNode and returns 1, otherwise
        # returns 0.
        module_scope = self.obj.analyse_as_module(env)
        if module_scope:
            entry = module_scope.lookup_here(self.attribute)
            if entry and (
                entry.is_cglobal or entry.is_cfunction
                or entry.is_type or entry.is_const):
                    self.mutate_into_name_node(env, entry, target)
4501
                    entry.used = 1
William Stein's avatar
William Stein committed
4502 4503
                    return 1
        return 0
4504

William Stein's avatar
William Stein committed
4505 4506 4507 4508 4509 4510 4511 4512 4513 4514 4515 4516 4517 4518 4519 4520
    def analyse_as_unbound_cmethod(self, env):
        # Try to interpret this as a reference to an unbound
        # C method of an extension type. If successful, mutates
        # this node into a NameNode and returns 1, otherwise
        # returns 0.
        type = self.obj.analyse_as_extension_type(env)
        if type:
            entry = type.scope.lookup_here(self.attribute)
            if entry and entry.is_cmethod:
                # Create a temporary entry describing the C method
                # as an ordinary function.
                ubcm_entry = Symtab.Entry(entry.name,
                    "%s->%s" % (type.vtabptr_cname, entry.cname),
                    entry.type)
                ubcm_entry.is_cfunction = 1
                ubcm_entry.func_cname = entry.func_cname
4521
                ubcm_entry.is_unbound_cmethod = 1
William Stein's avatar
William Stein committed
4522 4523 4524
                self.mutate_into_name_node(env, ubcm_entry, None)
                return 1
        return 0
4525

4526 4527 4528
    def analyse_as_type(self, env):
        module_scope = self.obj.analyse_as_module(env)
        if module_scope:
4529
            return module_scope.lookup_type(self.attribute)
4530
        if not self.obj.is_string_literal:
Robert Bradshaw's avatar
Robert Bradshaw committed
4531
            base_type = self.obj.analyse_as_type(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
4532
            if base_type and hasattr(base_type, 'scope') and base_type.scope is not None:
Robert Bradshaw's avatar
Robert Bradshaw committed
4533
                return base_type.scope.lookup_type(self.attribute)
4534
        return None
4535

William Stein's avatar
William Stein committed
4536 4537 4538 4539 4540 4541 4542 4543 4544
    def analyse_as_extension_type(self, env):
        # Try to interpret this as a reference to an extension type
        # in a cimported module. Returns the extension type, or None.
        module_scope = self.obj.analyse_as_module(env)
        if module_scope:
            entry = module_scope.lookup_here(self.attribute)
            if entry and entry.is_type and entry.type.is_extension_type:
                return entry.type
        return None
4545

William Stein's avatar
William Stein committed
4546 4547 4548 4549 4550 4551 4552 4553 4554
    def analyse_as_module(self, env):
        # Try to interpret this as a reference to a cimported module
        # in another cimported module. Returns the module scope, or None.
        module_scope = self.obj.analyse_as_module(env)
        if module_scope:
            entry = module_scope.lookup_here(self.attribute)
            if entry and entry.as_module:
                return entry.as_module
        return None
4555

William Stein's avatar
William Stein committed
4556 4557 4558 4559 4560 4561 4562 4563 4564 4565 4566
    def mutate_into_name_node(self, env, entry, target):
        # Mutate this node into a NameNode and complete the
        # analyse_types phase.
        self.__class__ = NameNode
        self.name = self.attribute
        self.entry = entry
        del self.obj
        del self.attribute
        if target:
            NameNode.analyse_target_types(self, env)
        else:
4567
            NameNode.analyse_rvalue_entry(self, env)
4568

William Stein's avatar
William Stein committed
4569 4570 4571 4572
    def analyse_as_ordinary_attribute(self, env, target):
        self.obj.analyse_types(env)
        self.analyse_attribute(env)
        if self.entry and self.entry.is_cmethod and not self.is_called:
4573 4574
#            error(self.pos, "C method can only be called")
            pass
4575 4576
        ## Reference to C array turns into pointer to first element.
        #while self.type.is_array:
Robert Bradshaw's avatar
Robert Bradshaw committed
4577
        #    self.type = self.type.element_ptr_type()
William Stein's avatar
William Stein committed
4578 4579 4580 4581
        if self.is_py_attr:
            if not target:
                self.is_temp = 1
                self.result_ctype = py_object_type
4582 4583
        elif target and self.obj.type.is_builtin_type:
            error(self.pos, "Assignment to an immutable object field")
4584 4585
        #elif self.type.is_memoryviewslice and not target:
        #    self.is_temp = True
4586

Robert Bradshaw's avatar
Robert Bradshaw committed
4587
    def analyse_attribute(self, env, obj_type = None):
William Stein's avatar
William Stein committed
4588
        # Look up attribute and set self.type and self.member.
4589
        immutable_obj = obj_type is not None # used during type inference
William Stein's avatar
William Stein committed
4590 4591
        self.is_py_attr = 0
        self.member = self.attribute
Robert Bradshaw's avatar
Robert Bradshaw committed
4592 4593 4594 4595 4596 4597 4598
        if obj_type is None:
            if self.obj.type.is_string:
                self.obj = self.obj.coerce_to_pyobject(env)
            obj_type = self.obj.type
        else:
            if obj_type.is_string:
                obj_type = py_object_type
4599
        if obj_type.is_ptr or obj_type.is_array:
William Stein's avatar
William Stein committed
4600 4601
            obj_type = obj_type.base_type
            self.op = "->"
4602
        elif obj_type.is_extension_type or obj_type.is_builtin_type:
William Stein's avatar
William Stein committed
4603 4604 4605 4606 4607 4608
            self.op = "->"
        else:
            self.op = "."
        if obj_type.has_attributes:
            entry = None
            if obj_type.attributes_known():
4609 4610
                if (obj_type.is_memoryviewslice and not
                        obj_type.scope.lookup_here(self.attribute)):
4611 4612 4613 4614 4615 4616 4617
                    if self.attribute == 'T':
                        self.is_memslice_transpose = True
                        self.is_temp = True
                        self.use_managed_ref = True
                        self.type = self.obj.type
                        return
                    else:
4618
                        obj_type.declare_attribute(self.attribute, env, self.pos)
William Stein's avatar
William Stein committed
4619
                entry = obj_type.scope.lookup_here(self.attribute)
Robert Bradshaw's avatar
Robert Bradshaw committed
4620 4621
                if entry and entry.is_member:
                    entry = None
William Stein's avatar
William Stein committed
4622
            else:
4623 4624
                error(self.pos,
                    "Cannot select attribute of incomplete type '%s'"
William Stein's avatar
William Stein committed
4625
                    % obj_type)
Robert Bradshaw's avatar
Robert Bradshaw committed
4626 4627
                self.type = PyrexTypes.error_type
                return
William Stein's avatar
William Stein committed
4628 4629
            self.entry = entry
            if entry:
4630 4631
                if obj_type.is_extension_type and entry.name == "__weakref__":
                    error(self.pos, "Illegal use of special attribute __weakref__")
4632 4633

                # def methods need the normal attribute lookup
4634
                # because they do not have struct entries
4635 4636 4637 4638
                # fused function go through assignment synthesis
                # (foo = pycfunction(foo_func_obj)) and need to go through
                # regular Python lookup as well
                if (entry.is_variable and not entry.fused_cfunction) or entry.is_cmethod:
4639 4640 4641
                    self.type = entry.type
                    self.member = entry.cname
                    return
William Stein's avatar
William Stein committed
4642 4643 4644 4645 4646
                else:
                    # If it's not a variable or C method, it must be a Python
                    # method of an extension type, so we treat it like a Python
                    # attribute.
                    pass
4647
        # If we get here, the base object is not a struct/union/extension
William Stein's avatar
William Stein committed
4648 4649 4650
        # type, or it is an extension type and the attribute is either not
        # declared or is declared as a Python method. Treat it as a Python
        # attribute reference.
4651
        self.analyse_as_python_attribute(env, obj_type, immutable_obj)
Stefan Behnel's avatar
Stefan Behnel committed
4652

4653
    def analyse_as_python_attribute(self, env, obj_type=None, immutable_obj=False):
Robert Bradshaw's avatar
Robert Bradshaw committed
4654 4655
        if obj_type is None:
            obj_type = self.obj.type
4656 4657
        # mangle private '__*' Python attributes used inside of a class
        self.attribute = env.mangle_class_private_name(self.attribute)
4658
        self.member = self.attribute
4659 4660
        self.type = py_object_type
        self.is_py_attr = 1
4661
        if not obj_type.is_pyobject and not obj_type.is_error:
4662
            if obj_type.can_coerce_to_pyobject(env):
4663 4664
                if not immutable_obj:
                    self.obj = self.obj.coerce_to_pyobject(env)
4665 4666 4667 4668
            elif (obj_type.is_cfunction and self.obj.is_name
                  and self.obj.entry.as_variable
                  and self.obj.entry.as_variable.type.is_pyobject):
                # might be an optimised builtin function => unpack it
4669 4670
                if not immutable_obj:
                    self.obj = self.obj.coerce_to_pyobject(env)
4671 4672 4673 4674
            else:
                error(self.pos,
                      "Object of type '%s' has no attribute '%s'" %
                      (obj_type, self.attribute))
4675

4676 4677 4678 4679 4680
    def wrap_obj_in_nonecheck(self, env):
        if not env.directives['nonecheck']:
            return

        msg = None
4681
        format_args = ()
4682 4683
        if (self.obj.type.is_extension_type and self.needs_none_check and not
                self.is_py_attr):
4684 4685
            msg = "'NoneType' object has no attribute '%s'"
            format_args = (self.attribute,)
4686 4687 4688 4689 4690 4691 4692
        elif self.obj.type.is_memoryviewslice:
            if self.is_memslice_transpose:
                msg = "Cannot transpose None memoryview slice"
            else:
                entry = self.obj.type.scope.lookup_here(self.attribute)
                if entry:
                    # copy/is_c_contig/shape/strides etc
4693 4694
                    msg = "Cannot access '%s' attribute of None memoryview slice"
                    format_args = (entry.name,)
4695 4696

        if msg:
4697 4698
            self.obj = self.obj.as_none_safe_node(msg, 'PyExc_AttributeError',
                                                  format_args=format_args)
4699 4700


4701
    def nogil_check(self, env):
4702
        if self.is_py_attr:
4703
            self.gil_error()
4704 4705 4706
        elif self.type.is_memoryviewslice:
            import MemoryView
            MemoryView.err_if_nogil_initialized_check(self.pos, env, 'attribute')
4707

4708 4709
    gil_message = "Accessing Python attribute"

William Stein's avatar
William Stein committed
4710 4711 4712 4713 4714 4715 4716 4717
    def is_simple(self):
        if self.obj:
            return self.result_in_temp() or self.obj.is_simple()
        else:
            return NameNode.is_simple(self)

    def is_lvalue(self):
        if self.obj:
4718
            return not self.type.is_array
William Stein's avatar
William Stein committed
4719 4720
        else:
            return NameNode.is_lvalue(self)
4721

William Stein's avatar
William Stein committed
4722 4723 4724 4725 4726
    def is_ephemeral(self):
        if self.obj:
            return self.obj.is_ephemeral()
        else:
            return NameNode.is_ephemeral(self)
4727

William Stein's avatar
William Stein committed
4728 4729
    def calculate_result_code(self):
        #print "AttributeNode.calculate_result_code:", self.member ###
4730
        #print "...obj node =", self.obj, "code", self.obj.result() ###
William Stein's avatar
William Stein committed
4731 4732 4733 4734 4735
        #print "...obj type", self.obj.type, "ctype", self.obj.ctype() ###
        obj = self.obj
        obj_code = obj.result_as(obj.type)
        #print "...obj_code =", obj_code ###
        if self.entry and self.entry.is_cmethod:
Stefan Behnel's avatar
Stefan Behnel committed
4736
            if obj.type.is_extension_type and not self.entry.is_builtin_cmethod:
4737 4738
                if self.entry.final_func_cname:
                    return self.entry.final_func_cname
4739

Mark Florisson's avatar
Mark Florisson committed
4740
                if self.type.from_fused:
4741 4742 4743 4744
                    # If the attribute was specialized through indexing, make
                    # sure to get the right fused name, as our entry was
                    # replaced by our parent index node
                    # (AnalyseExpressionsTransform)
Mark Florisson's avatar
Mark Florisson committed
4745 4746
                    self.member = self.entry.cname

Robert Bradshaw's avatar
Robert Bradshaw committed
4747
                return "((struct %s *)%s%s%s)->%s" % (
4748
                    obj.type.vtabstruct_cname, obj_code, self.op,
Robert Bradshaw's avatar
Robert Bradshaw committed
4749 4750 4751
                    obj.type.vtabslot_cname, self.member)
            else:
                return self.member
4752
        elif obj.type.is_complex:
4753
            return "__Pyx_C%s(%s)" % (self.member.upper(), obj_code)
William Stein's avatar
William Stein committed
4754
        else:
4755 4756 4757
            if obj.type.is_builtin_type and self.entry and self.entry.is_variable:
                # accessing a field of a builtin type, need to cast better than result_as() does
                obj_code = obj.type.cast_code(obj.result(), to_object_struct = True)
William Stein's avatar
William Stein committed
4758
            return "%s%s%s" % (obj_code, self.op, self.member)
4759

William Stein's avatar
William Stein committed
4760 4761
    def generate_result_code(self, code):
        if self.is_py_attr:
4762 4763
            code.putln(
                '%s = PyObject_GetAttr(%s, %s); %s' % (
4764
                    self.result(),
4765
                    self.obj.py_result(),
4766
                    code.intern_identifier(self.attribute),
4767
                    code.error_goto_if_null(self.result(), self.pos)))
4768
            code.put_gotref(self.py_result())
4769
        elif self.type.is_memoryviewslice:
4770 4771 4772 4773 4774 4775 4776 4777 4778
            if self.is_memslice_transpose:
                # transpose the slice
                for access, packing in self.type.axes:
                    if access == 'ptr':
                        error(self.pos, "Transposing not supported for slices "
                                        "with indirect dimensions")
                        return

                code.putln("%s = %s;" % (self.result(), self.obj.result()))
4779 4780
                if self.obj.is_name or (self.obj.is_attribute and
                                        self.obj.is_memslice_transpose):
4781 4782 4783 4784 4785
                    code.put_incref_memoryviewslice(self.result(), have_gil=True)

                T = "__pyx_memslice_transpose(&%s) == 0"
                code.putln(code.error_goto_if(T % self.result(), self.pos))
            elif self.initialized_check:
4786 4787 4788 4789 4790 4791
                code.putln(
                    'if (unlikely(!%s.memview)) {'
                        'PyErr_SetString(PyExc_AttributeError,'
                                        '"Memoryview is not initialized");'
                        '%s'
                    '}' % (self.result(), code.error_goto(self.pos)))
4792 4793 4794
        else:
            # result_code contains what is needed, but we may need to insert
            # a check and raise an exception
4795
            if self.obj.type.is_extension_type:
4796
                pass
4797 4798 4799
            elif self.entry and self.entry.is_cmethod and self.entry.utility_code:
                # C method implemented as function call with utility code
                code.globalstate.use_utility_code(self.entry.utility_code)
4800

William Stein's avatar
William Stein committed
4801 4802 4803
    def generate_assignment_code(self, rhs, code):
        self.obj.generate_evaluation_code(code)
        if self.is_py_attr:
4804
            code.put_error_if_neg(self.pos,
4805 4806
                'PyObject_SetAttr(%s, %s, %s)' % (
                    self.obj.py_result(),
4807
                    code.intern_identifier(self.attribute),
4808
                    rhs.py_result()))
William Stein's avatar
William Stein committed
4809
            rhs.generate_disposal_code(code)
4810
            rhs.free_temps(code)
4811 4812 4813 4814 4815
        elif self.obj.type.is_complex:
            code.putln("__Pyx_SET_C%s(%s, %s);" % (
                self.member.upper(),
                self.obj.result_as(self.obj.type),
                rhs.result_as(self.ctype())))
William Stein's avatar
William Stein committed
4816
        else:
4817
            select_code = self.result()
4818
            if self.type.is_pyobject and self.use_managed_ref:
William Stein's avatar
William Stein committed
4819
                rhs.make_owned_reference(code)
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
4820
                code.put_giveref(rhs.py_result())
4821
                code.put_gotref(select_code)
William Stein's avatar
William Stein committed
4822
                code.put_decref(select_code, self.ctype())
4823
            elif self.type.is_memoryviewslice:
4824
                import MemoryView
4825
                MemoryView.put_assign_to_memviewslice(
4826
                        select_code, rhs, rhs.result(), self.type, code)
4827

4828
            if not self.type.is_memoryviewslice:
4829 4830 4831 4832 4833
                code.putln(
                    "%s = %s;" % (
                        select_code,
                        rhs.result_as(self.ctype())))
                        #rhs.result()))
William Stein's avatar
William Stein committed
4834
            rhs.generate_post_assignment_code(code)
4835
            rhs.free_temps(code)
William Stein's avatar
William Stein committed
4836
        self.obj.generate_disposal_code(code)
4837
        self.obj.free_temps(code)
4838

William Stein's avatar
William Stein committed
4839 4840
    def generate_deletion_code(self, code):
        self.obj.generate_evaluation_code(code)
4841
        if self.is_py_attr or (isinstance(self.entry.scope, Symtab.PropertyScope)
4842
                               and u'__del__' in self.entry.scope.entries):
4843 4844 4845
            code.put_error_if_neg(self.pos,
                'PyObject_DelAttr(%s, %s)' % (
                    self.obj.py_result(),
4846
                    code.intern_identifier(self.attribute)))
William Stein's avatar
William Stein committed
4847 4848 4849
        else:
            error(self.pos, "Cannot delete C attribute of extension type")
        self.obj.generate_disposal_code(code)
4850
        self.obj.free_temps(code)
4851

4852 4853 4854 4855 4856
    def annotate(self, code):
        if self.is_py_attr:
            code.annotate(self.pos, AnnotationItem('py_attr', 'python attribute', size=len(self.attribute)))
        else:
            code.annotate(self.pos, AnnotationItem('c_attr', 'c attribute', size=len(self.attribute)))
William Stein's avatar
William Stein committed
4857

4858

William Stein's avatar
William Stein committed
4859 4860 4861 4862 4863 4864
#-------------------------------------------------------------------
#
#  Constructor nodes
#
#-------------------------------------------------------------------

4865 4866 4867 4868 4869 4870 4871 4872 4873 4874 4875 4876 4877 4878 4879
class StarredTargetNode(ExprNode):
    #  A starred expression like "*a"
    #
    #  This is only allowed in sequence assignment targets such as
    #
    #      a, *b = (1,2,3,4)    =>     a = 1 ; b = [2,3,4]
    #
    #  and will be removed during type analysis (or generate an error
    #  if it's found at unexpected places).
    #
    #  target          ExprNode

    subexprs = ['target']
    is_starred = 1
    type = py_object_type
Robert Bradshaw's avatar
Robert Bradshaw committed
4880
    is_temp = 1
4881 4882 4883 4884 4885 4886 4887 4888 4889 4890 4891 4892 4893 4894 4895 4896 4897 4898 4899 4900 4901 4902 4903 4904 4905 4906 4907 4908

    def __init__(self, pos, target):
        self.pos = pos
        self.target = target

    def analyse_declarations(self, env):
        error(self.pos, "can use starred expression only as assignment target")
        self.target.analyse_declarations(env)

    def analyse_types(self, env):
        error(self.pos, "can use starred expression only as assignment target")
        self.target.analyse_types(env)
        self.type = self.target.type

    def analyse_target_declaration(self, env):
        self.target.analyse_target_declaration(env)

    def analyse_target_types(self, env):
        self.target.analyse_target_types(env)
        self.type = self.target.type

    def calculate_result_code(self):
        return ""

    def generate_result_code(self, code):
        pass


4909
class SequenceNode(ExprNode):
William Stein's avatar
William Stein committed
4910 4911 4912 4913 4914 4915
    #  Base class for list and tuple constructor nodes.
    #  Contains common code for performing sequence unpacking.
    #
    #  args                    [ExprNode]
    #  unpacked_items          [ExprNode] or None
    #  coerced_unpacked_items  [ExprNode] or None
4916
    # mult_factor              ExprNode     the integer number of content repetitions ([1,2]*3)
4917

4918
    subexprs = ['args', 'mult_factor']
4919

William Stein's avatar
William Stein committed
4920 4921
    is_sequence_constructor = 1
    unpacked_items = None
4922
    mult_factor = None
4923

4924 4925 4926
    def compile_time_value_list(self, denv):
        return [arg.compile_time_value(denv) for arg in self.args]

4927 4928 4929 4930 4931 4932 4933 4934 4935 4936 4937 4938 4939 4940
    def replace_starred_target_node(self):
        # replace a starred node in the targets by the contained expression
        self.starred_assignment = False
        args = []
        for arg in self.args:
            if arg.is_starred:
                if self.starred_assignment:
                    error(arg.pos, "more than 1 starred expression in assignment")
                self.starred_assignment = True
                arg = arg.target
                arg.is_starred = True
            args.append(arg)
        self.args = args

William Stein's avatar
William Stein committed
4941
    def analyse_target_declaration(self, env):
4942
        self.replace_starred_target_node()
William Stein's avatar
William Stein committed
4943 4944 4945
        for arg in self.args:
            arg.analyse_target_declaration(env)

4946
    def analyse_types(self, env, skip_children=False):
William Stein's avatar
William Stein committed
4947 4948
        for i in range(len(self.args)):
            arg = self.args[i]
4949
            if not skip_children: arg.analyse_types(env)
William Stein's avatar
William Stein committed
4950
            self.args[i] = arg.coerce_to_pyobject(env)
4951 4952 4953
        if self.mult_factor:
            self.mult_factor.analyse_types(env)
            if not self.mult_factor.type.is_int:
4954
                self.mult_factor = self.mult_factor.coerce_to_pyobject(env)
William Stein's avatar
William Stein committed
4955
        self.is_temp = 1
Stefan Behnel's avatar
Stefan Behnel committed
4956
        # not setting self.type here, subtypes do this
4957

4958 4959 4960
    def may_be_none(self):
        return False

William Stein's avatar
William Stein committed
4961
    def analyse_target_types(self, env):
4962
        if self.mult_factor:
Stefan Behnel's avatar
Stefan Behnel committed
4963
            error(self.pos, "can't assign to multiplied sequence")
4964
        self.unpacked_items = []
William Stein's avatar
William Stein committed
4965
        self.coerced_unpacked_items = []
4966
        self.any_coerced_items = False
William Stein's avatar
William Stein committed
4967 4968
        for arg in self.args:
            arg.analyse_target_types(env)
4969 4970 4971 4972 4973 4974
            if arg.is_starred:
                if not arg.type.assignable_from(Builtin.list_type):
                    error(arg.pos,
                          "starred target must have Python object (list) type")
                if arg.type is py_object_type:
                    arg.type = Builtin.list_type
William Stein's avatar
William Stein committed
4975 4976
            unpacked_item = PyTempNode(self.pos, env)
            coerced_unpacked_item = unpacked_item.coerce_to(arg.type, env)
4977 4978
            if unpacked_item is not coerced_unpacked_item:
                self.any_coerced_items = True
William Stein's avatar
William Stein committed
4979 4980 4981
            self.unpacked_items.append(unpacked_item)
            self.coerced_unpacked_items.append(coerced_unpacked_item)
        self.type = py_object_type
4982

William Stein's avatar
William Stein committed
4983 4984
    def generate_result_code(self, code):
        self.generate_operation_code(code)
4985

4986 4987 4988
    def generate_sequence_packing_code(self, code, target=None, plain=False):
        if target is None:
            target = self.result()
4989 4990 4991 4992
        size_factor = c_mult = ''
        mult_factor = None

        if self.mult_factor and not plain:
4993
            mult_factor = self.mult_factor
4994 4995 4996 4997 4998 4999 5000
            if mult_factor.type.is_int:
                c_mult = mult_factor.result()
                if isinstance(mult_factor.constant_result, (int,long)) \
                       and mult_factor.constant_result > 0:
                    size_factor = ' * %s' % mult_factor.constant_result
                else:
                    size_factor = ' * ((%s<0) ? 0:%s)' % (c_mult, c_mult)
5001

5002 5003 5004 5005 5006 5007 5008 5009
        if self.type is Builtin.tuple_type and self.is_literal and not c_mult:
            # use PyTuple_Pack() to avoid generating huge amounts of one-time code
            code.putln('%s = PyTuple_Pack(%d, %s); %s' % (
                target,
                len(self.args),
                ', '.join([ arg.py_result() for arg in self.args ]),
                code.error_goto_if_null(target, self.pos)))
            code.put_gotref(target)
5010
        else:
5011 5012 5013 5014 5015
            # build the tuple/list step by step, potentially multiplying it as we go
            if self.type is Builtin.list_type:
                create_func, set_item_func = 'PyList_New', 'PyList_SET_ITEM'
            elif self.type is Builtin.tuple_type:
                create_func, set_item_func = 'PyTuple_New', 'PyTuple_SET_ITEM'
5016
            else:
5017 5018 5019 5020 5021 5022 5023 5024 5025 5026 5027 5028 5029 5030 5031 5032 5033 5034 5035 5036 5037 5038 5039 5040 5041 5042 5043 5044 5045 5046 5047 5048 5049 5050 5051 5052 5053 5054 5055 5056 5057 5058
                raise InternalError("sequence packing for unexpected type %s" % self.type)
            arg_count = len(self.args)
            code.putln("%s = %s(%s%s); %s" % (
                target, create_func, arg_count, size_factor,
                code.error_goto_if_null(target, self.pos)))
            code.put_gotref(target)

            if c_mult:
                # FIXME: can't use a temp variable here as the code may
                # end up in the constant building function.  Temps
                # currently don't work there.

                #counter = code.funcstate.allocate_temp(mult_factor.type, manage_ref=False)
                counter = Naming.quick_temp_cname
                code.putln('{ Py_ssize_t %s;' % counter)
                if arg_count == 1:
                    offset = counter
                else:
                    offset = '%s * %s' % (counter, arg_count)
                code.putln('for (%s=0; %s < %s; %s++) {' % (
                    counter, counter, c_mult, counter
                    ))
            else:
                offset = ''

            for i in xrange(arg_count):
                arg = self.args[i]
                if c_mult or not arg.result_in_temp():
                    code.put_incref(arg.result(), arg.ctype())
                code.putln("%s(%s, %s, %s);" % (
                    set_item_func,
                    target,
                    (offset and i) and ('%s + %s' % (offset, i)) or (offset or i),
                    arg.py_result()))
                code.put_giveref(arg.py_result())

            if c_mult:
                code.putln('}')
                #code.funcstate.release_temp(counter)
                code.putln('}')

        if mult_factor is not None and mult_factor.type.is_pyobject:
5059
            code.putln('{ PyObject* %s = PyNumber_InPlaceMultiply(%s, %s); %s' % (
5060
                Naming.quick_temp_cname, target, mult_factor.py_result(),
5061 5062 5063 5064 5065 5066
                code.error_goto_if_null(Naming.quick_temp_cname, self.pos)
                ))
            code.put_gotref(Naming.quick_temp_cname)
            code.put_decref(target, py_object_type)
            code.putln('%s = %s;' % (target, Naming.quick_temp_cname))
            code.putln('}')
5067 5068

    def generate_subexpr_disposal_code(self, code):
5069
        if self.mult_factor and self.mult_factor.type.is_int:
5070 5071 5072 5073 5074 5075 5076 5077 5078
            super(SequenceNode, self).generate_subexpr_disposal_code(code)
        else:
            # We call generate_post_assignment_code here instead
            # of generate_disposal_code, because values were stored
            # in the tuple using a reference-stealing operation.
            for arg in self.args:
                arg.generate_post_assignment_code(code)
                # Should NOT call free_temps -- this is invoked by the default
                # generate_evaluation_code which will do that.
5079 5080
            if self.mult_factor:
                self.mult_factor.generate_disposal_code(code)
5081

William Stein's avatar
William Stein committed
5082
    def generate_assignment_code(self, rhs, code):
5083 5084 5085
        if self.starred_assignment:
            self.generate_starred_assignment_code(rhs, code)
        else:
5086
            self.generate_parallel_assignment_code(rhs, code)
5087 5088 5089 5090 5091

        for item in self.unpacked_items:
            item.release(code)
        rhs.free_temps(code)

5092 5093 5094 5095 5096
    _func_iternext_type = PyrexTypes.CPtrType(PyrexTypes.CFuncType(
        PyrexTypes.py_object_type, [
            PyrexTypes.CFuncTypeArg("it", PyrexTypes.py_object_type, None),
            ]))

5097
    def generate_parallel_assignment_code(self, rhs, code):
5098 5099 5100
        # Need to work around the fact that generate_evaluation_code
        # allocates the temps in a rather hacky way -- the assignment
        # is evaluated twice, within each if-block.
5101 5102
        for item in self.unpacked_items:
            item.allocate(code)
5103 5104 5105
        special_unpack = (rhs.type is py_object_type
                          or rhs.type in (tuple_type, list_type)
                          or not rhs.type.is_builtin_type)
5106
        long_enough_for_a_loop = len(self.unpacked_items) > 3
5107

5108
        if special_unpack:
5109 5110
            self.generate_special_parallel_unpacking_code(
                code, rhs, use_loop=long_enough_for_a_loop)
5111
        code.putln("{")
Stefan Behnel's avatar
Stefan Behnel committed
5112 5113
        self.generate_generic_parallel_unpacking_code(
            code, rhs, self.unpacked_items, use_loop=long_enough_for_a_loop)
5114
        code.putln("}")
5115

5116 5117
        for value_node in self.coerced_unpacked_items:
            value_node.generate_evaluation_code(code)
5118 5119 5120
        for i in range(len(self.args)):
            self.args[i].generate_assignment_code(
                self.coerced_unpacked_items[i], code)
5121

5122
    def generate_special_parallel_unpacking_code(self, code, rhs, use_loop):
5123 5124 5125 5126 5127 5128 5129 5130 5131 5132 5133 5134 5135 5136
        tuple_check = 'likely(PyTuple_CheckExact(%s))' % rhs.py_result()
        list_check  = 'PyList_CheckExact(%s)' % rhs.py_result()
        sequence_type_test = '1'
        if rhs.type is list_type:
            sequence_types = ['List']
            if rhs.may_be_none():
                sequence_type_test = list_check
        elif rhs.type is tuple_type:
            sequence_types = ['Tuple']
            if rhs.may_be_none():
                sequence_type_test = tuple_check
        else:
            sequence_types = ['Tuple', 'List']
            sequence_type_test = "(%s) || (%s)" % (tuple_check, list_check)
5137

5138 5139
        code.putln("if (%s) {" % sequence_type_test)
        code.putln("PyObject* sequence = %s;" % rhs.py_result())
5140

Stefan Behnel's avatar
Stefan Behnel committed
5141
        # list/tuple => check size
5142
        code.putln("#if CYTHON_COMPILING_IN_CPYTHON")
5143
        code.putln("Py_ssize_t size = Py_SIZE(sequence);")
5144 5145 5146
        code.putln("#else")
        code.putln("Py_ssize_t size = PySequence_Size(sequence);")  # < 0 => exception
        code.putln("#endif")
5147 5148 5149 5150 5151
        code.putln("if (unlikely(size != %d)) {" % len(self.args))
        code.globalstate.use_utility_code(raise_too_many_values_to_unpack)
        code.putln("if (size > %d) __Pyx_RaiseTooManyValuesError(%d);" % (
            len(self.args), len(self.args)))
        code.globalstate.use_utility_code(raise_need_more_values_to_unpack)
5152
        code.putln("else if (size >= 0) __Pyx_RaiseNeedMoreValuesError(size);")
5153 5154
        code.putln(code.error_goto(self.pos))
        code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
5155

5156
        code.putln("#if CYTHON_COMPILING_IN_CPYTHON")
5157 5158 5159 5160 5161 5162 5163 5164
        # unpack items from list/tuple in unrolled loop (can't fail)
        if len(sequence_types) == 2:
            code.putln("if (likely(Py%s_CheckExact(sequence))) {" % sequence_types[0])
        for i, item in enumerate(self.unpacked_items):
            code.putln("%s = Py%s_GET_ITEM(sequence, %d); " % (
                item.result(), sequence_types[0], i))
        if len(sequence_types) == 2:
            code.putln("} else {")
5165
            for i, item in enumerate(self.unpacked_items):
5166 5167 5168 5169 5170
                code.putln("%s = Py%s_GET_ITEM(sequence, %d); " % (
                    item.result(), sequence_types[1], i))
            code.putln("}")
        for item in self.unpacked_items:
            code.put_incref(item.result(), item.ctype())
5171 5172 5173 5174 5175 5176 5177 5178

        code.putln("#else")
        # in non-CPython, use the PySequence protocol (which can fail)
        if not use_loop:
            for i, item in enumerate(self.unpacked_items):
                code.putln("%s = PySequence_ITEM(sequence, %d); %s" % (
                    item.result(), i,
                    code.error_goto_if_null(item.result(), self.pos)))
5179
                code.put_gotref(item.result())
5180 5181 5182 5183 5184 5185 5186 5187
        else:
            code.putln("Py_ssize_t i;")
            code.putln("PyObject** temps[%s] = {%s};" % (
                len(self.unpacked_items),
                ','.join(['&%s' % item.result() for item in self.unpacked_items])))
            code.putln("for (i=0; i < %s; i++) {" % len(self.unpacked_items))
            code.putln("PyObject* item = PySequence_ITEM(sequence, i); %s" % (
                code.error_goto_if_null('item', self.pos)))
5188
            code.put_gotref('item')
5189 5190 5191 5192
            code.putln("*(temps[i]) = item;")
            code.putln("}")

        code.putln("#endif")
5193 5194 5195 5196 5197 5198 5199 5200 5201
        rhs.generate_disposal_code(code)

        if rhs.type is tuple_type:
            # if not a tuple: None => save some code by generating the error directly
            code.putln("} else if (1) {")
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("RaiseNoneIterError", "ObjectHandling.c"))
            code.putln("__Pyx_RaiseNoneNotIterableError(); %s" % code.error_goto(self.pos))
        code.putln("} else")
5202

5203
    def generate_generic_parallel_unpacking_code(self, code, rhs, unpacked_items, use_loop, terminate=True):
5204
        code.globalstate.use_utility_code(raise_need_more_values_to_unpack)
5205
        code.globalstate.use_utility_code(UtilityCode.load_cached("IterFinish", "ObjectHandling.c"))
5206
        code.putln("Py_ssize_t index = -1;") # must be at the start of a C block!
5207

5208 5209 5210
        if use_loop:
            code.putln("PyObject** temps[%s] = {%s};" % (
                len(self.unpacked_items),
5211
                ','.join(['&%s' % item.result() for item in unpacked_items])))
5212

5213 5214 5215 5216 5217 5218 5219 5220
        iterator_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
        code.putln(
            "%s = PyObject_GetIter(%s); %s" % (
                iterator_temp,
                rhs.py_result(),
                code.error_goto_if_null(iterator_temp, self.pos)))
        code.put_gotref(iterator_temp)
        rhs.generate_disposal_code(code)
5221

5222 5223 5224
        iternext_func = code.funcstate.allocate_temp(self._func_iternext_type, manage_ref=False)
        code.putln("%s = Py_TYPE(%s)->tp_iternext;" % (
            iternext_func, iterator_temp))
William Stein's avatar
William Stein committed
5225

5226 5227
        unpacking_error_label = code.new_label('unpacking_failed')
        unpack_code = "%s(%s)" % (iternext_func, iterator_temp)
5228
        if use_loop:
5229
            code.putln("for (index=0; index < %s; index++) {" % len(unpacked_items))
5230 5231 5232 5233 5234 5235
            code.put("PyObject* item = %s; if (unlikely(!item)) " % unpack_code)
            code.put_goto(unpacking_error_label)
            code.put_gotref("item")
            code.putln("*(temps[index]) = item;")
            code.putln("}")
        else:
5236
            for i, item in enumerate(unpacked_items):
5237 5238 5239 5240 5241 5242 5243 5244
                code.put(
                    "index = %d; %s = %s; if (unlikely(!%s)) " % (
                        i,
                        item.result(),
                        unpack_code,
                        item.result()))
                code.put_goto(unpacking_error_label)
                code.put_gotref(item.py_result())
5245 5246

        if terminate:
5247 5248
            code.globalstate.use_utility_code(
                UtilityCode.load_cached("UnpackItemEndCheck", "ObjectHandling.c"))
5249 5250 5251 5252 5253 5254
            code.put_error_if_neg(self.pos, "__Pyx_IternextUnpackEndCheck(%s, %d)" % (
                unpack_code,
                len(unpacked_items)))
            code.putln("%s = NULL;" % iternext_func)
            code.put_decref_clear(iterator_temp, py_object_type)

5255 5256 5257 5258 5259
        unpacking_done_label = code.new_label('unpacking_done')
        code.put_goto(unpacking_done_label)

        code.put_label(unpacking_error_label)
        code.put_decref_clear(iterator_temp, py_object_type)
5260
        code.putln("%s = NULL;" % iternext_func)
5261
        code.putln("if (__Pyx_IterFinish() == 0) __Pyx_RaiseNeedMoreValuesError(index);")
5262 5263
        code.putln(code.error_goto(self.pos))
        code.put_label(unpacking_done_label)
5264

5265 5266 5267 5268 5269 5270 5271
        code.funcstate.release_temp(iternext_func)
        if terminate:
            code.funcstate.release_temp(iterator_temp)
            iterator_temp = None

        return iterator_temp

5272 5273 5274 5275
    def generate_starred_assignment_code(self, rhs, code):
        for i, arg in enumerate(self.args):
            if arg.is_starred:
                starred_target = self.unpacked_items[i]
5276 5277
                unpacked_fixed_items_left  = self.unpacked_items[:i]
                unpacked_fixed_items_right = self.unpacked_items[i+1:]
5278
                break
5279 5280
        else:
            assert False
5281

5282 5283
        iterator_temp = None
        if unpacked_fixed_items_left:
5284
            for item in unpacked_fixed_items_left:
5285
                item.allocate(code)
5286 5287 5288 5289 5290
            code.putln('{')
            iterator_temp = self.generate_generic_parallel_unpacking_code(
                code, rhs, unpacked_fixed_items_left,
                use_loop=True, terminate=False)
            for i, item in enumerate(unpacked_fixed_items_left):
5291 5292
                value_node = self.coerced_unpacked_items[i]
                value_node.generate_evaluation_code(code)
5293
            code.putln('}')
5294

5295
        starred_target.allocate(code)
5296 5297
        target_list = starred_target.result()
        code.putln("%s = PySequence_List(%s); %s" % (
5298 5299
            target_list,
            iterator_temp or rhs.py_result(),
5300 5301
            code.error_goto_if_null(target_list, self.pos)))
        code.put_gotref(target_list)
5302 5303 5304 5305 5306 5307 5308 5309

        if iterator_temp:
            code.put_decref_clear(iterator_temp, py_object_type)
            code.funcstate.release_temp(iterator_temp)
        else:
            rhs.generate_disposal_code(code)

        if unpacked_fixed_items_right:
5310
            code.globalstate.use_utility_code(raise_need_more_values_to_unpack)
5311 5312 5313 5314 5315 5316
            length_temp = code.funcstate.allocate_temp(PyrexTypes.c_py_ssize_t_type, manage_ref=False)
            code.putln('%s = PyList_GET_SIZE(%s);' % (length_temp, target_list))
            code.putln("if (unlikely(%s < %d)) {" % (length_temp, len(unpacked_fixed_items_right)))
            code.putln("__Pyx_RaiseNeedMoreValuesError(%d+%s); %s" % (
                 len(unpacked_fixed_items_left), length_temp,
                 code.error_goto(self.pos)))
5317
            code.putln('}')
5318 5319 5320 5321 5322

            for item in unpacked_fixed_items_right[::-1]:
                item.allocate(code)
            for i, (item, coerced_arg) in enumerate(zip(unpacked_fixed_items_right[::-1],
                                                        self.coerced_unpacked_items[::-1])):
5323 5324 5325
                code.putln('#if CYTHON_COMPILING_IN_CPYTHON')
                code.putln("%s = PyList_GET_ITEM(%s, %s-%d); " % (
                    item.py_result(), target_list, length_temp, i+1))
5326
                # resize the list the hard way
5327
                code.putln("((PyVarObject*)%s)->ob_size--;" % target_list)
5328
                code.putln('#else')
5329
                code.putln("%s = PySequence_ITEM(%s, %s-%d); " % (
5330 5331
                    item.py_result(), target_list, length_temp, i+1))
                code.putln('#endif')
5332
                code.put_gotref(item.py_result())
5333 5334
                coerced_arg.generate_evaluation_code(code)

5335
            code.putln('#if !CYTHON_COMPILING_IN_CPYTHON')
5336
            sublist_temp = code.funcstate.allocate_temp(py_object_type, manage_ref=True)
Stefan Behnel's avatar
Stefan Behnel committed
5337
            code.putln('%s = PySequence_GetSlice(%s, 0, %s-%d); %s' % (
5338 5339 5340 5341 5342 5343
                sublist_temp, target_list, length_temp, len(unpacked_fixed_items_right),
                code.error_goto_if_null(sublist_temp, self.pos)))
            code.put_gotref(sublist_temp)
            code.funcstate.release_temp(length_temp)
            code.put_decref(target_list, py_object_type)
            code.putln('%s = %s; %s = NULL;' % (target_list, sublist_temp, sublist_temp))
5344 5345
            code.putln('#else')
            code.putln('%s = %s;' % (sublist_temp, sublist_temp)) # avoid warning about unused variable
5346 5347 5348 5349 5350
            code.funcstate.release_temp(sublist_temp)
            code.putln('#endif')

        for i, arg in enumerate(self.args):
            arg.generate_assignment_code(self.coerced_unpacked_items[i], code)
5351

5352 5353 5354 5355 5356 5357 5358 5359
    def annotate(self, code):
        for arg in self.args:
            arg.annotate(code)
        if self.unpacked_items:
            for arg in self.unpacked_items:
                arg.annotate(code)
            for arg in self.coerced_unpacked_items:
                arg.annotate(code)
William Stein's avatar
William Stein committed
5360 5361 5362 5363


class TupleNode(SequenceNode):
    #  Tuple constructor.
5364

5365
    type = tuple_type
5366
    is_partly_literal = False
5367 5368 5369

    gil_message = "Constructing Python tuple"

5370
    def analyse_types(self, env, skip_children=False):
Robert Bradshaw's avatar
Robert Bradshaw committed
5371
        if len(self.args) == 0:
5372 5373
            self.is_temp = False
            self.is_literal = True
Robert Bradshaw's avatar
Robert Bradshaw committed
5374
        else:
5375
            SequenceNode.analyse_types(self, env, skip_children)
5376 5377 5378 5379
            for child in self.args:
                if not child.is_literal:
                    break
            else:
5380 5381 5382 5383 5384
                if not self.mult_factor or self.mult_factor.is_literal and \
                       isinstance(self.mult_factor.constant_result, (int, long)):
                    self.is_temp = False
                    self.is_literal = True
                else:
5385 5386
                    if not self.mult_factor.type.is_pyobject:
                        self.mult_factor = self.mult_factor.coerce_to_pyobject(env)
5387 5388
                    self.is_temp = True
                    self.is_partly_literal = True
5389

Stefan Behnel's avatar
Stefan Behnel committed
5390 5391 5392 5393
    def is_simple(self):
        # either temp or constant => always simple
        return True

5394 5395 5396 5397
    def nonlocally_immutable(self):
        # either temp or constant => always safe
        return True

Robert Bradshaw's avatar
Robert Bradshaw committed
5398 5399
    def calculate_result_code(self):
        if len(self.args) > 0:
5400
            return self.result_code
Robert Bradshaw's avatar
Robert Bradshaw committed
5401 5402
        else:
            return Naming.empty_tuple
William Stein's avatar
William Stein committed
5403

5404 5405 5406 5407
    def calculate_constant_result(self):
        self.constant_result = tuple([
                arg.constant_result for arg in self.args])

5408 5409 5410 5411 5412 5413
    def compile_time_value(self, denv):
        values = self.compile_time_value_list(denv)
        try:
            return tuple(values)
        except Exception, e:
            self.compile_time_value_error(e)
5414

William Stein's avatar
William Stein committed
5415
    def generate_operation_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
5416 5417 5418
        if len(self.args) == 0:
            # result_code is Naming.empty_tuple
            return
5419 5420 5421 5422 5423 5424 5425 5426 5427 5428 5429 5430 5431
        if self.is_partly_literal:
            # underlying tuple is const, but factor is not
            tuple_target = code.get_py_const(py_object_type, 'tuple_', cleanup_level=2)
            const_code = code.get_cached_constants_writer()
            const_code.mark_pos(self.pos)
            self.generate_sequence_packing_code(const_code, tuple_target, plain=True)
            const_code.put_giveref(tuple_target)
            code.putln('%s = PyNumber_Multiply(%s, %s); %s' % (
                self.result(), tuple_target, self.mult_factor.py_result(),
                code.error_goto_if_null(self.result(), self.pos)
                ))
            code.put_gotref(self.py_result())
        elif self.is_literal:
5432 5433
            # non-empty cached tuple => result is global constant,
            # creation code goes into separate code writer
5434
            self.result_code = code.get_py_const(py_object_type, 'tuple_', cleanup_level=2)
5435 5436
            code = code.get_cached_constants_writer()
            code.mark_pos(self.pos)
5437
            self.generate_sequence_packing_code(code)
5438
            code.put_giveref(self.py_result())
5439 5440
        else:
            self.generate_sequence_packing_code(code)
William Stein's avatar
William Stein committed
5441 5442 5443 5444


class ListNode(SequenceNode):
    #  List constructor.
5445

5446 5447
    # obj_conversion_errors    [PyrexError]   used internally
    # orignial_args            [ExprNode]     used internally
5448

5449
    obj_conversion_errors = []
Stefan Behnel's avatar
Stefan Behnel committed
5450
    type = list_type
5451

5452
    gil_message = "Constructing Python list"
5453

Robert Bradshaw's avatar
Robert Bradshaw committed
5454
    def type_dependencies(self, env):
5455
        return ()
5456

5457 5458 5459
    def infer_type(self, env):
        # TOOD: Infer non-object list arrays.
        return list_type
5460

5461
    def analyse_expressions(self, env):
5462
        SequenceNode.analyse_expressions(self, env)
5463 5464
        self.coerce_to_pyobject(env)

Robert Bradshaw's avatar
Robert Bradshaw committed
5465
    def analyse_types(self, env):
5466 5467 5468 5469 5470
        hold_errors()
        self.original_args = list(self.args)
        SequenceNode.analyse_types(self, env)
        self.obj_conversion_errors = held_errors()
        release_errors(ignore=True)
5471

Robert Bradshaw's avatar
Robert Bradshaw committed
5472 5473
    def coerce_to(self, dst_type, env):
        if dst_type.is_pyobject:
5474 5475 5476
            for err in self.obj_conversion_errors:
                report_error(err)
            self.obj_conversion_errors = []
Robert Bradshaw's avatar
Robert Bradshaw committed
5477 5478
            if not self.type.subtype_of(dst_type):
                error(self.pos, "Cannot coerce list to type '%s'" % dst_type)
5479 5480
        elif self.mult_factor:
            error(self.pos, "Cannot coerce multiplied list to '%s'" % dst_type)
5481
        elif dst_type.is_ptr and dst_type.base_type is not PyrexTypes.c_void_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
5482
            base_type = dst_type.base_type
Robert Bradshaw's avatar
Robert Bradshaw committed
5483
            self.type = PyrexTypes.CArrayType(base_type, len(self.args))
5484
            for i in range(len(self.original_args)):
Robert Bradshaw's avatar
Robert Bradshaw committed
5485
                arg = self.args[i]
5486 5487
                if isinstance(arg, CoerceToPyTypeNode):
                    arg = arg.arg
Robert Bradshaw's avatar
Robert Bradshaw committed
5488
                self.args[i] = arg.coerce_to(base_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5489 5490 5491 5492 5493 5494
        elif dst_type.is_struct:
            if len(self.args) > len(dst_type.scope.var_entries):
                error(self.pos, "Too may members for '%s'" % dst_type)
            else:
                if len(self.args) < len(dst_type.scope.var_entries):
                    warning(self.pos, "Too few members for '%s'" % dst_type, 1)
5495 5496 5497
                for i, (arg, member) in enumerate(zip(self.original_args, dst_type.scope.var_entries)):
                    if isinstance(arg, CoerceToPyTypeNode):
                        arg = arg.arg
Robert Bradshaw's avatar
Robert Bradshaw committed
5498 5499
                    self.args[i] = arg.coerce_to(member.type, env)
            self.type = dst_type
Robert Bradshaw's avatar
Robert Bradshaw committed
5500 5501 5502 5503
        else:
            self.type = error_type
            error(self.pos, "Cannot coerce list to type '%s'" % dst_type)
        return self
5504

5505
    def release_temp_result(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
5506
        if self.type.is_array:
5507 5508
            # To be valid C++, we must allocate the memory on the stack
            # manually and be sure not to reuse it for something else.
Robert Bradshaw's avatar
Robert Bradshaw committed
5509 5510
            pass
        else:
5511
            SequenceNode.release_temp_result(self, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
5512

5513
    def calculate_constant_result(self):
5514 5515
        if self.mult_factor:
            raise ValueError() # may exceed the compile time memory
5516 5517 5518
        self.constant_result = [
            arg.constant_result for arg in self.args]

5519
    def compile_time_value(self, denv):
5520 5521 5522 5523
        l = self.compile_time_value_list(denv)
        if self.mult_factor:
            l *= self.mult_factor.compile_time_value(denv)
        return l
5524

William Stein's avatar
William Stein committed
5525
    def generate_operation_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
5526
        if self.type.is_pyobject:
5527 5528
            for err in self.obj_conversion_errors:
                report_error(err)
5529
            self.generate_sequence_packing_code(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
5530 5531 5532 5533 5534 5535
        elif self.type.is_array:
            for i, arg in enumerate(self.args):
                code.putln("%s[%s] = %s;" % (
                                self.result(),
                                i,
                                arg.result()))
Robert Bradshaw's avatar
Robert Bradshaw committed
5536
        elif self.type.is_struct:
Robert Bradshaw's avatar
Robert Bradshaw committed
5537 5538 5539 5540 5541
            for arg, member in zip(self.args, self.type.scope.var_entries):
                code.putln("%s.%s = %s;" % (
                        self.result(),
                        member.cname,
                        arg.result()))
5542 5543
        else:
            raise InternalError("List type never specified")
5544

Robert Bradshaw's avatar
Robert Bradshaw committed
5545

5546 5547 5548 5549 5550 5551 5552 5553 5554
class ScopedExprNode(ExprNode):
    # Abstract base class for ExprNodes that have their own local
    # scope, such as generator expressions.
    #
    # expr_scope    Scope  the inner scope of the expression

    subexprs = []
    expr_scope = None

5555 5556 5557 5558 5559 5560 5561 5562 5563 5564 5565 5566 5567 5568 5569 5570 5571 5572 5573
    # does this node really have a local scope, e.g. does it leak loop
    # variables or not?  non-leaking Py3 behaviour is default, except
    # for list comprehensions where the behaviour differs in Py2 and
    # Py3 (set in Parsing.py based on parser context)
    has_local_scope = True

    def init_scope(self, outer_scope, expr_scope=None):
        if expr_scope is not None:
            self.expr_scope = expr_scope
        elif self.has_local_scope:
            self.expr_scope = Symtab.GeneratorExpressionScope(outer_scope)
        else:
            self.expr_scope = None

    def analyse_declarations(self, env):
        self.init_scope(env)

    def analyse_scoped_declarations(self, env):
        # this is called with the expr_scope as env
5574 5575
        pass

5576 5577
    def analyse_types(self, env):
        # no recursion here, the children will be analysed separately below
5578 5579 5580 5581 5582 5583
        pass

    def analyse_scoped_expressions(self, env):
        # this is called with the expr_scope as env
        pass

5584 5585 5586 5587 5588 5589 5590 5591 5592 5593 5594 5595 5596 5597 5598 5599 5600 5601 5602 5603 5604 5605 5606 5607 5608 5609 5610 5611 5612 5613 5614 5615 5616 5617 5618 5619 5620 5621 5622 5623 5624 5625 5626 5627 5628 5629 5630
    def generate_evaluation_code(self, code):
        # set up local variables and free their references on exit
        generate_inner_evaluation_code = super(ScopedExprNode, self).generate_evaluation_code
        if not self.has_local_scope or not self.expr_scope.var_entries:
            # no local variables => delegate, done
            generate_inner_evaluation_code(code)
            return

        code.putln('{ /* enter inner scope */')
        py_entries = []
        for entry in self.expr_scope.var_entries:
            if not entry.in_closure:
                code.put_var_declaration(entry)
                if entry.type.is_pyobject and entry.used:
                    py_entries.append(entry)
        if not py_entries:
            # no local Python references => no cleanup required
            generate_inner_evaluation_code(code)
            code.putln('} /* exit inner scope */')
            return

        # must free all local Python references at each exit point
        old_loop_labels = tuple(code.new_loop_labels())
        old_error_label = code.new_error_label()

        generate_inner_evaluation_code(code)

        # normal (non-error) exit
        for entry in py_entries:
            code.put_var_decref(entry)

        # error/loop body exit points
        exit_scope = code.new_label('exit_scope')
        code.put_goto(exit_scope)
        for label, old_label in ([(code.error_label, old_error_label)] +
                                 list(zip(code.get_loop_labels(), old_loop_labels))):
            if code.label_used(label):
                code.put_label(label)
                for entry in py_entries:
                    code.put_var_decref(entry)
                code.put_goto(old_label)
        code.put_label(exit_scope)
        code.putln('} /* exit inner scope */')

        code.set_loop_labels(old_loop_labels)
        code.error_label = old_error_label

5631 5632

class ComprehensionNode(ScopedExprNode):
5633
    subexprs = ["target"]
5634
    child_attrs = ["loop"]
5635

5636 5637
    def infer_type(self, env):
        return self.target.infer_type(env)
5638 5639 5640

    def analyse_declarations(self, env):
        self.append.target = self # this is used in the PyList_Append of the inner loop
5641 5642
        self.init_scope(env)

5643 5644
    def analyse_scoped_declarations(self, env):
        self.loop.analyse_declarations(env)
5645

5646 5647 5648
    def analyse_types(self, env):
        self.target.analyse_expressions(env)
        self.type = self.target.type
5649 5650
        if not self.has_local_scope:
            self.loop.analyse_expressions(env)
5651

5652 5653 5654
    def analyse_scoped_expressions(self, env):
        if self.has_local_scope:
            self.loop.analyse_expressions(env)
5655

5656 5657 5658
    def may_be_none(self):
        return False

5659 5660
    def calculate_result_code(self):
        return self.target.result()
5661

5662 5663
    def generate_result_code(self, code):
        self.generate_operation_code(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
5664

5665 5666 5667
    def generate_operation_code(self, code):
        self.loop.generate_execution_code(code)

5668 5669
    def annotate(self, code):
        self.loop.annotate(code)
5670 5671


5672
class ComprehensionAppendNode(Node):
5673 5674
    # Need to be careful to avoid infinite recursion:
    # target must not be in child_attrs/subexprs
5675 5676

    child_attrs = ['expr']
5677 5678

    type = PyrexTypes.c_int_type
5679

5680 5681
    def analyse_expressions(self, env):
        self.expr.analyse_expressions(env)
5682
        if not self.expr.type.is_pyobject:
Robert Bradshaw's avatar
Robert Bradshaw committed
5683
            self.expr = self.expr.coerce_to_pyobject(env)
5684

5685
    def generate_execution_code(self, code):
5686
        if self.target.type is list_type:
5687 5688
            code.globalstate.use_utility_code(UtilityCode.load_cached("InternalListAppend", "Optimize.c"))
            function = "__Pyx_PyList_Append"
5689 5690 5691 5692 5693
        elif self.target.type is set_type:
            function = "PySet_Add"
        else:
            raise InternalError(
                "Invalid type for comprehension node: %s" % self.target.type)
5694 5695 5696 5697 5698 5699 5700 5701 5702 5703 5704 5705 5706 5707 5708

        self.expr.generate_evaluation_code(code)
        code.putln(code.error_goto_if("%s(%s, (PyObject*)%s)" % (
            function,
            self.target.result(),
            self.expr.result()
            ), self.pos))
        self.expr.generate_disposal_code(code)
        self.expr.free_temps(code)

    def generate_function_definitions(self, env, code):
        self.expr.generate_function_definitions(env, code)

    def annotate(self, code):
        self.expr.annotate(code)
5709 5710

class DictComprehensionAppendNode(ComprehensionAppendNode):
5711
    child_attrs = ['key_expr', 'value_expr']
5712

5713 5714
    def analyse_expressions(self, env):
        self.key_expr.analyse_expressions(env)
5715 5716
        if not self.key_expr.type.is_pyobject:
            self.key_expr = self.key_expr.coerce_to_pyobject(env)
5717
        self.value_expr.analyse_expressions(env)
5718 5719 5720
        if not self.value_expr.type.is_pyobject:
            self.value_expr = self.value_expr.coerce_to_pyobject(env)

5721 5722 5723 5724 5725 5726 5727 5728 5729 5730 5731 5732 5733 5734 5735 5736 5737 5738 5739 5740
    def generate_execution_code(self, code):
        self.key_expr.generate_evaluation_code(code)
        self.value_expr.generate_evaluation_code(code)
        code.putln(code.error_goto_if("PyDict_SetItem(%s, (PyObject*)%s, (PyObject*)%s)" % (
            self.target.result(),
            self.key_expr.result(),
            self.value_expr.result()
            ), self.pos))
        self.key_expr.generate_disposal_code(code)
        self.key_expr.free_temps(code)
        self.value_expr.generate_disposal_code(code)
        self.value_expr.free_temps(code)

    def generate_function_definitions(self, env, code):
        self.key_expr.generate_function_definitions(env, code)
        self.value_expr.generate_function_definitions(env, code)

    def annotate(self, code):
        self.key_expr.annotate(code)
        self.value_expr.annotate(code)
5741 5742


5743 5744 5745 5746 5747
class InlinedGeneratorExpressionNode(ScopedExprNode):
    # An inlined generator expression for which the result is
    # calculated inside of the loop.  This will only be created by
    # transforms when replacing builtin calls on generator
    # expressions.
5748
    #
5749 5750 5751
    # loop           ForStatNode      the for-loop, not containing any YieldExprNodes
    # result_node    ResultRefNode    the reference to the result value temp
    # orig_func      String           the name of the builtin function this node replaces
5752

5753
    child_attrs = ["loop"]
5754
    loop_analysed = False
5755 5756
    type = py_object_type

5757 5758
    def analyse_scoped_declarations(self, env):
        self.loop.analyse_declarations(env)
5759

5760 5761 5762 5763 5764 5765
    def may_be_none(self):
        return False

    def annotate(self, code):
        self.loop.annotate(code)

5766 5767
    def infer_type(self, env):
        return self.result_node.infer_type(env)
5768 5769

    def analyse_types(self, env):
5770 5771 5772
        if not self.has_local_scope:
            self.loop_analysed = True
            self.loop.analyse_expressions(env)
5773 5774 5775
        self.type = self.result_node.type
        self.is_temp = True

5776 5777
    def analyse_scoped_expressions(self, env):
        self.loop_analysed = True
5778 5779
        if self.has_local_scope:
            self.loop.analyse_expressions(env)
5780

5781
    def coerce_to(self, dst_type, env):
5782 5783 5784 5785 5786 5787
        if self.orig_func == 'sum' and dst_type.is_numeric and not self.loop_analysed:
            # We can optimise by dropping the aggregation variable and
            # the add operations into C.  This can only be done safely
            # before analysing the loop body, after that, the result
            # reference type will have infected expressions and
            # assignments.
5788 5789
            self.result_node.type = self.type = dst_type
            return self
5790
        return super(InlinedGeneratorExpressionNode, self).coerce_to(dst_type, env)
5791

5792 5793 5794 5795 5796
    def generate_result_code(self, code):
        self.result_node.result_code = self.result()
        self.loop.generate_execution_code(code)


5797
class SetNode(ExprNode):
5798 5799
    #  Set constructor.

5800 5801
    type = set_type

5802 5803 5804
    subexprs = ['args']

    gil_message = "Constructing Python set"
5805

5806 5807 5808 5809 5810 5811 5812 5813
    def analyse_types(self, env):
        for i in range(len(self.args)):
            arg = self.args[i]
            arg.analyse_types(env)
            self.args[i] = arg.coerce_to_pyobject(env)
        self.type = set_type
        self.is_temp = 1

5814 5815 5816
    def may_be_none(self):
        return False

5817 5818 5819 5820
    def calculate_constant_result(self):
        self.constant_result = set([
                arg.constant_result for arg in self.args])

5821 5822 5823 5824 5825 5826 5827 5828
    def compile_time_value(self, denv):
        values = [arg.compile_time_value(denv) for arg in self.args]
        try:
            return set(values)
        except Exception, e:
            self.compile_time_value_error(e)

    def generate_evaluation_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
5829
        code.globalstate.use_utility_code(Builtin.py_set_utility_code)
5830 5831 5832 5833 5834
        self.allocate_temp_result(code)
        code.putln(
            "%s = PySet_New(0); %s" % (
                self.result(),
                code.error_goto_if_null(self.result(), self.pos)))
5835
        code.put_gotref(self.py_result())
5836 5837 5838 5839 5840 5841 5842 5843
        for arg in self.args:
            arg.generate_evaluation_code(code)
            code.putln(
                code.error_goto_if_neg(
                    "PySet_Add(%s, %s)" % (self.result(), arg.py_result()),
                    self.pos))
            arg.generate_disposal_code(code)
            arg.free_temps(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
5844

William Stein's avatar
William Stein committed
5845

5846
class DictNode(ExprNode):
William Stein's avatar
William Stein committed
5847 5848
    #  Dictionary constructor.
    #
Vitja Makarov's avatar
Vitja Makarov committed
5849 5850
    #  key_value_pairs     [DictItemNode]
    #  exclude_null_values [boolean]          Do not add NULL values to dict
5851 5852
    #
    # obj_conversion_errors    [PyrexError]   used internally
5853

5854
    subexprs = ['key_value_pairs']
5855
    is_temp = 1
Vitja Makarov's avatar
Vitja Makarov committed
5856
    exclude_null_values = False
5857
    type = dict_type
5858

5859
    obj_conversion_errors = []
5860

5861 5862 5863 5864 5865
    @classmethod
    def from_pairs(cls, pos, pairs):
        return cls(pos, key_value_pairs=[
                DictItemNode(pos, key=k, value=v) for k, v in pairs])

5866 5867 5868
    def calculate_constant_result(self):
        self.constant_result = dict([
                item.constant_result for item in self.key_value_pairs])
5869

5870
    def compile_time_value(self, denv):
Robert Bradshaw's avatar
Robert Bradshaw committed
5871 5872
        pairs = [(item.key.compile_time_value(denv), item.value.compile_time_value(denv))
            for item in self.key_value_pairs]
5873 5874 5875 5876
        try:
            return dict(pairs)
        except Exception, e:
            self.compile_time_value_error(e)
5877

Robert Bradshaw's avatar
Robert Bradshaw committed
5878
    def type_dependencies(self, env):
5879
        return ()
5880

5881 5882 5883 5884
    def infer_type(self, env):
        # TOOD: Infer struct constructors.
        return dict_type

William Stein's avatar
William Stein committed
5885
    def analyse_types(self, env):
5886
        hold_errors()
Robert Bradshaw's avatar
Robert Bradshaw committed
5887 5888
        for item in self.key_value_pairs:
            item.analyse_types(env)
5889 5890
        self.obj_conversion_errors = held_errors()
        release_errors(ignore=True)
5891 5892 5893

    def may_be_none(self):
        return False
5894

5895 5896 5897 5898 5899 5900 5901 5902 5903 5904 5905 5906 5907 5908
    def coerce_to(self, dst_type, env):
        if dst_type.is_pyobject:
            self.release_errors()
            if not self.type.subtype_of(dst_type):
                error(self.pos, "Cannot interpret dict as type '%s'" % dst_type)
        elif dst_type.is_struct_or_union:
            self.type = dst_type
            if not dst_type.is_struct and len(self.key_value_pairs) != 1:
                error(self.pos, "Exactly one field must be specified to convert to union '%s'" % dst_type)
            elif dst_type.is_struct and len(self.key_value_pairs) < len(dst_type.scope.var_entries):
                warning(self.pos, "Not all members given for struct '%s'" % dst_type, 1)
            for item in self.key_value_pairs:
                if isinstance(item.key, CoerceToPyTypeNode):
                    item.key = item.key.arg
5909
                if not item.key.is_string_literal:
5910
                    error(item.key.pos, "Invalid struct field identifier")
5911
                    item.key = StringNode(item.key.pos, value="<error>")
5912
                else:
Stefan Behnel's avatar
Stefan Behnel committed
5913 5914
                    key = str(item.key.value) # converts string literals to unicode in Py3
                    member = dst_type.scope.lookup_here(key)
5915
                    if not member:
Stefan Behnel's avatar
Stefan Behnel committed
5916
                        error(item.key.pos, "struct '%s' has no field '%s'" % (dst_type, key))
5917 5918 5919 5920 5921 5922 5923 5924 5925
                    else:
                        value = item.value
                        if isinstance(value, CoerceToPyTypeNode):
                            value = value.arg
                        item.value = value.coerce_to(member.type, env)
        else:
            self.type = error_type
            error(self.pos, "Cannot interpret dict as type '%s'" % dst_type)
        return self
5926

5927 5928 5929 5930
    def release_errors(self):
        for err in self.obj_conversion_errors:
            report_error(err)
        self.obj_conversion_errors = []
5931 5932 5933

    gil_message = "Constructing Python dict"

William Stein's avatar
William Stein committed
5934 5935 5936
    def generate_evaluation_code(self, code):
        #  Custom method used here because key-value
        #  pairs are evaluated and used one at a time.
5937 5938
        code.mark_pos(self.pos)
        self.allocate_temp_result(code)
5939 5940 5941 5942 5943 5944
        if self.type.is_pyobject:
            self.release_errors()
            code.putln(
                "%s = PyDict_New(); %s" % (
                    self.result(),
                    code.error_goto_if_null(self.result(), self.pos)))
5945
            code.put_gotref(self.py_result())
Robert Bradshaw's avatar
Robert Bradshaw committed
5946 5947
        for item in self.key_value_pairs:
            item.generate_evaluation_code(code)
5948
            if self.type.is_pyobject:
Vitja Makarov's avatar
Vitja Makarov committed
5949 5950
                if self.exclude_null_values:
                    code.putln('if (%s) {' % item.value.py_result())
5951
                code.put_error_if_neg(self.pos,
5952 5953 5954 5955
                    "PyDict_SetItem(%s, %s, %s)" % (
                        self.result(),
                        item.key.py_result(),
                        item.value.py_result()))
Vitja Makarov's avatar
Vitja Makarov committed
5956 5957
                if self.exclude_null_values:
                    code.putln('}')
5958 5959 5960
            else:
                code.putln("%s.%s = %s;" % (
                        self.result(),
5961
                        item.key.value,
5962
                        item.value.result()))
Robert Bradshaw's avatar
Robert Bradshaw committed
5963
            item.generate_disposal_code(code)
5964
            item.free_temps(code)
5965

5966
    def annotate(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
5967 5968
        for item in self.key_value_pairs:
            item.annotate(code)
5969

5970
class DictItemNode(ExprNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
5971 5972 5973 5974 5975
    # Represents a single item in a DictNode
    #
    # key          ExprNode
    # value        ExprNode
    subexprs = ['key', 'value']
5976

5977
    nogil_check = None # Parent DictNode takes care of it
5978

5979 5980 5981
    def calculate_constant_result(self):
        self.constant_result = (
            self.key.constant_result, self.value.constant_result)
5982

Robert Bradshaw's avatar
Robert Bradshaw committed
5983 5984 5985 5986 5987
    def analyse_types(self, env):
        self.key.analyse_types(env)
        self.value.analyse_types(env)
        self.key = self.key.coerce_to_pyobject(env)
        self.value = self.value.coerce_to_pyobject(env)
5988

Robert Bradshaw's avatar
Robert Bradshaw committed
5989 5990 5991
    def generate_evaluation_code(self, code):
        self.key.generate_evaluation_code(code)
        self.value.generate_evaluation_code(code)
Stefan Behnel's avatar
Stefan Behnel committed
5992

5993 5994 5995
    def generate_disposal_code(self, code):
        self.key.generate_disposal_code(code)
        self.value.generate_disposal_code(code)
5996 5997 5998 5999

    def free_temps(self, code):
        self.key.free_temps(code)
        self.value.free_temps(code)
6000

6001 6002
    def __iter__(self):
        return iter([self.key, self.value])
William Stein's avatar
William Stein committed
6003

6004

6005 6006 6007 6008 6009 6010 6011
class ModuleNameMixin(object):
    def set_mod_name(self, env):
        self.module_name = env.global_scope().qualified_name

    def get_py_mod_name(self, code):
        return code.get_py_string_const(
                 self.module_name, identifier=True)
Stefan Behnel's avatar
Stefan Behnel committed
6012

6013
class ClassNode(ExprNode, ModuleNameMixin):
William Stein's avatar
William Stein committed
6014 6015 6016 6017
    #  Helper class used in the implementation of Python
    #  class definitions. Constructs a class object given
    #  a name, tuple of bases and class dictionary.
    #
Stefan Behnel's avatar
Stefan Behnel committed
6018
    #  name         EncodedString      Name of the class
William Stein's avatar
William Stein committed
6019 6020 6021
    #  bases        ExprNode           Base class tuple
    #  dict         ExprNode           Class dict (not owned by this node)
    #  doc          ExprNode or None   Doc string
6022
    #  module_name  EncodedString      Name of defining module
6023

6024
    subexprs = ['bases', 'doc']
6025

William Stein's avatar
William Stein committed
6026 6027 6028 6029 6030 6031 6032
    def analyse_types(self, env):
        self.bases.analyse_types(env)
        if self.doc:
            self.doc.analyse_types(env)
            self.doc = self.doc.coerce_to_pyobject(env)
        self.type = py_object_type
        self.is_temp = 1
6033
        env.use_utility_code(UtilityCode.load_cached("CreateClass", "ObjectHandling.c"))
6034 6035
        #TODO(craig,haoyu) This should be moved to a better place
        self.set_mod_name(env)
6036

6037
    def may_be_none(self):
Stefan Behnel's avatar
Stefan Behnel committed
6038
        return True
6039

6040 6041
    gil_message = "Constructing Python class"

William Stein's avatar
William Stein committed
6042
    def generate_result_code(self, code):
6043
        cname = code.intern_identifier(self.name)
6044

William Stein's avatar
William Stein committed
6045
        if self.doc:
6046
            code.put_error_if_neg(self.pos,
Robert Bradshaw's avatar
Robert Bradshaw committed
6047
                'PyDict_SetItemString(%s, "__doc__", %s)' % (
William Stein's avatar
William Stein committed
6048
                    self.dict.py_result(),
Robert Bradshaw's avatar
Robert Bradshaw committed
6049
                    self.doc.py_result()))
6050
        py_mod_name = self.get_py_mod_name(code)
William Stein's avatar
William Stein committed
6051
        code.putln(
6052
            '%s = __Pyx_CreateClass(%s, %s, %s, %s); %s' % (
6053
                self.result(),
William Stein's avatar
William Stein committed
6054 6055
                self.bases.py_result(),
                self.dict.py_result(),
6056
                cname,
6057
                py_mod_name,
6058
                code.error_goto_if_null(self.result(), self.pos)))
6059
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
6060

Stefan Behnel's avatar
Stefan Behnel committed
6061

6062 6063 6064 6065 6066 6067 6068 6069 6070 6071 6072 6073 6074 6075 6076 6077 6078 6079
class Py3ClassNode(ExprNode):
    #  Helper class used in the implementation of Python3+
    #  class definitions. Constructs a class object given
    #  a name, tuple of bases and class dictionary.
    #
    #  name         EncodedString      Name of the class
    #  dict         ExprNode           Class dict (not owned by this node)
    #  module_name  EncodedString      Name of defining module

    subexprs = []

    def analyse_types(self, env):
        self.type = py_object_type
        self.is_temp = 1

    def may_be_none(self):
        return True

6080
    gil_message = "Constructing Python class"
6081 6082

    def generate_result_code(self, code):
6083
        code.globalstate.use_utility_code(UtilityCode.load_cached("Py3ClassCreate", "ObjectHandling.c"))
6084 6085 6086 6087 6088 6089 6090 6091 6092 6093 6094 6095 6096
        cname = code.intern_identifier(self.name)
        code.putln(
            '%s = __Pyx_Py3ClassCreate(%s, %s, %s, %s, %s); %s' % (
                self.result(),
                self.metaclass.result(),
                cname,
                self.bases.py_result(),
                self.dict.py_result(),
                self.mkw.py_result(),
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())

class KeywordArgsNode(ExprNode):
6097
    #  Helper class for keyword arguments.
6098
    #
6099 6100
    #  starstar_arg      DictNode
    #  keyword_args      [DictItemNode]
6101

6102 6103 6104 6105 6106 6107 6108 6109 6110 6111 6112 6113 6114 6115 6116 6117 6118 6119 6120 6121 6122 6123 6124 6125 6126 6127
    subexprs = ['starstar_arg', 'keyword_args']
    is_temp = 1
    type = dict_type

    def calculate_constant_result(self):
        result = dict(self.starstar_arg.constant_result)
        for item in self.keyword_args:
            key, value = item.constant_result
            if key in result:
                raise ValueError("duplicate keyword argument found: %s" % key)
            result[key] = value
        self.constant_result = result

    def compile_time_value(self, denv):
        result = self.starstar_arg.compile_time_value(denv)
        pairs = [ (item.key.compile_time_value(denv), item.value.compile_time_value(denv))
                  for item in self.keyword_args ]
        try:
            result = dict(result)
            for key, value in pairs:
                if key in result:
                    raise ValueError("duplicate keyword argument found: %s" % key)
                result[key] = value
        except Exception, e:
            self.compile_time_value_error(e)
        return result
6128

6129 6130 6131 6132 6133
    def type_dependencies(self, env):
        return ()

    def infer_type(self, env):
        return dict_type
6134 6135

    def analyse_types(self, env):
6136 6137 6138 6139 6140 6141
        self.starstar_arg.analyse_types(env)
        self.starstar_arg = self.starstar_arg.coerce_to_pyobject(env).as_none_safe_node(
            # FIXME: CPython's error message starts with the runtime function name
            'argument after ** must be a mapping, not NoneType')
        for item in self.keyword_args:
            item.analyse_types(env)
6142

6143 6144
    def may_be_none(self):
        return False
6145

6146 6147 6148 6149 6150 6151 6152 6153 6154 6155
    gil_message = "Constructing Python dict"

    def generate_evaluation_code(self, code):
        code.mark_pos(self.pos)
        self.allocate_temp_result(code)
        self.starstar_arg.generate_evaluation_code(code)
        if self.starstar_arg.type is not Builtin.dict_type:
            # CPython supports calling functions with non-dicts, so do we
            code.putln('if (likely(PyDict_Check(%s))) {' %
                       self.starstar_arg.py_result())
6156 6157 6158 6159 6160 6161
        if self.keyword_args:
            code.putln(
                "%s = PyDict_Copy(%s); %s" % (
                    self.result(),
                    self.starstar_arg.py_result(),
                    code.error_goto_if_null(self.result(), self.pos)))
6162
            code.put_gotref(self.py_result())
6163
        else:
6164 6165 6166 6167 6168 6169
            code.putln("%s = %s;" % (
                self.result(),
                self.starstar_arg.py_result()))
            code.put_incref(self.result(), py_object_type)
        if self.starstar_arg.type is not Builtin.dict_type:
            code.putln('} else {')
6170
            code.putln(
6171 6172
                "%s = PyObject_CallFunctionObjArgs("
                "(PyObject*)&PyDict_Type, %s, NULL); %s" % (
6173
                    self.result(),
6174
                    self.starstar_arg.py_result(),
6175
                    code.error_goto_if_null(self.result(), self.pos)))
6176
            code.put_gotref(self.py_result())
6177 6178 6179 6180 6181 6182 6183
            code.putln('}')
        self.starstar_arg.generate_disposal_code(code)
        self.starstar_arg.free_temps(code)

        if not self.keyword_args:
            return

6184 6185
        code.globalstate.use_utility_code(
            UtilityCode.load_cached("RaiseDoubleKeywords", "FunctionArguments.c"))
6186 6187 6188 6189 6190 6191 6192 6193 6194 6195 6196 6197 6198 6199 6200 6201 6202 6203 6204 6205 6206 6207
        for item in self.keyword_args:
            item.generate_evaluation_code(code)
            code.putln("if (unlikely(PyDict_GetItem(%s, %s))) {" % (
                    self.result(),
                    item.key.py_result()))
            # FIXME: find out function name at runtime!
            code.putln('__Pyx_RaiseDoubleKeywordsError("function", %s); %s' % (
                item.key.py_result(),
                code.error_goto(self.pos)))
            code.putln("}")
            code.put_error_if_neg(self.pos,
                "PyDict_SetItem(%s, %s, %s)" % (
                    self.result(),
                    item.key.py_result(),
                    item.value.py_result()))
            item.generate_disposal_code(code)
            item.free_temps(code)

    def annotate(self, code):
        self.starstar_arg.annotate(code)
        for item in self.keyword_args:
            item.annotate(code)
6208 6209 6210 6211 6212 6213 6214 6215 6216 6217 6218 6219 6220 6221 6222 6223 6224

class PyClassMetaclassNode(ExprNode):
    # Helper class holds Python3 metaclass object
    #
    #  bases        ExprNode           Base class tuple (not owned by this node)
    #  mkw          ExprNode           Class keyword arguments (not owned by this node)

    subexprs = []

    def analyse_types(self, env):
        self.type = py_object_type
        self.is_temp = True

    def may_be_none(self):
        return True

    def generate_result_code(self, code):
6225
        code.globalstate.use_utility_code(UtilityCode.load_cached("Py3MetaclassGet", "ObjectHandling.c"))
6226 6227 6228 6229 6230 6231 6232 6233 6234 6235 6236 6237 6238 6239 6240 6241 6242 6243 6244 6245 6246 6247 6248 6249 6250 6251 6252 6253 6254 6255 6256 6257 6258 6259 6260 6261 6262 6263 6264 6265 6266 6267 6268 6269 6270 6271 6272 6273 6274 6275 6276
        code.putln(
            "%s = __Pyx_Py3MetaclassGet(%s, %s); %s" % (
                self.result(),
                self.bases.result(),
                self.mkw.result(),
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())

class PyClassNamespaceNode(ExprNode, ModuleNameMixin):
    # Helper class holds Python3 namespace object
    #
    # All this are not owned by this node
    #  metaclass    ExprNode           Metaclass object
    #  bases        ExprNode           Base class tuple
    #  mkw          ExprNode           Class keyword arguments
    #  doc          ExprNode or None   Doc string (owned)

    subexprs = ['doc']

    def analyse_types(self, env):
        self.bases.analyse_types(env)
        if self.doc:
            self.doc.analyse_types(env)
            self.doc = self.doc.coerce_to_pyobject(env)
        self.type = py_object_type
        self.is_temp = 1
        #TODO(craig,haoyu) This should be moved to a better place
        self.set_mod_name(env)

    def may_be_none(self):
        return True

    def generate_result_code(self, code):
        cname = code.intern_identifier(self.name)
        py_mod_name = self.get_py_mod_name(code)
        if self.doc:
            doc_code = self.doc.result()
        else:
            doc_code = '(PyObject *) NULL'
        code.putln(
            "%s = __Pyx_Py3MetaclassPrepare(%s, %s, %s, %s, %s, %s); %s" % (
                self.result(),
                self.metaclass.result(),
                self.bases.result(),
                cname,
                self.mkw.result(),
                py_mod_name,
                doc_code,
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())

6277 6278 6279 6280 6281 6282 6283 6284 6285 6286

class ClassCellInjectorNode(ExprNode):
    # Initialize CyFunction.func_classobj
    is_temp = True
    type = py_object_type
    subexprs = []
    is_active = False

    def analyse_expressions(self, env):
        if self.is_active:
Stefan Behnel's avatar
Stefan Behnel committed
6287 6288
            env.use_utility_code(
                UtilityCode.load_cached("CyFunctionClassCell", "CythonFunction.c"))
6289 6290 6291 6292 6293 6294 6295 6296 6297 6298 6299 6300 6301 6302 6303 6304 6305 6306 6307 6308 6309 6310 6311 6312 6313 6314 6315 6316 6317 6318 6319 6320 6321

    def generate_evaluation_code(self, code):
        if self.is_active:
            self.allocate_temp_result(code)
            code.putln(
                '%s = PyList_New(0); %s' % (
                    self.result(),
                    code.error_goto_if_null(self.result(), self.pos)))
            code.put_gotref(self.result())

    def generate_injection_code(self, code, classobj_cname):
        if self.is_active:
            code.putln('__Pyx_CyFunction_InitClassCell(%s, %s);' % (
                self.result(), classobj_cname))


class ClassCellNode(ExprNode):
    # Class Cell for noargs super()
    subexprs = []
    is_temp = True
    is_generator = False
    type = py_object_type

    def analyse_types(self, env):
        pass

    def generate_result_code(self, code):
        if not self.is_generator:
            code.putln('%s = __Pyx_CyFunction_GetClassObj(%s);' % (
                self.result(),
                Naming.self_cname))
        else:
            code.putln('%s =  %s->classobj;' % (
6322
                self.result(), Naming.generator_cname))
6323 6324 6325 6326 6327 6328 6329 6330
        code.putln(
            'if (!%s) { PyErr_SetString(PyExc_SystemError, '
            '"super(): empty __class__ cell"); %s }' % (
                self.result(),
                code.error_goto(self.pos)));
        code.put_incref(self.result(), py_object_type)


Robert Bradshaw's avatar
Robert Bradshaw committed
6331 6332 6333 6334 6335 6336 6337
class BoundMethodNode(ExprNode):
    #  Helper class used in the implementation of Python
    #  class definitions. Constructs an bound method
    #  object from a class and a function.
    #
    #  function      ExprNode   Function object
    #  self_object   ExprNode   self object
6338

Robert Bradshaw's avatar
Robert Bradshaw committed
6339
    subexprs = ['function']
6340

Robert Bradshaw's avatar
Robert Bradshaw committed
6341 6342 6343 6344 6345 6346 6347 6348 6349 6350 6351 6352 6353 6354 6355 6356
    def analyse_types(self, env):
        self.function.analyse_types(env)
        self.type = py_object_type
        self.is_temp = 1

    gil_message = "Constructing an bound method"

    def generate_result_code(self, code):
        code.putln(
            "%s = PyMethod_New(%s, %s, (PyObject*)%s->ob_type); %s" % (
                self.result(),
                self.function.py_result(),
                self.self_object.py_result(),
                self.self_object.py_result(),
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
6357

6358
class UnboundMethodNode(ExprNode):
William Stein's avatar
William Stein committed
6359 6360 6361 6362 6363
    #  Helper class used in the implementation of Python
    #  class definitions. Constructs an unbound method
    #  object from a class and a function.
    #
    #  function      ExprNode   Function object
6364

6365 6366
    type = py_object_type
    is_temp = 1
6367

William Stein's avatar
William Stein committed
6368
    subexprs = ['function']
6369

William Stein's avatar
William Stein committed
6370 6371
    def analyse_types(self, env):
        self.function.analyse_types(env)
6372

6373 6374 6375
    def may_be_none(self):
        return False

6376 6377
    gil_message = "Constructing an unbound method"

William Stein's avatar
William Stein committed
6378
    def generate_result_code(self, code):
6379
        class_cname = code.pyclass_stack[-1].classobj.result()
William Stein's avatar
William Stein committed
6380
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
6381
            "%s = PyMethod_New(%s, 0, %s); %s" % (
6382
                self.result(),
William Stein's avatar
William Stein committed
6383
                self.function.py_result(),
6384
                class_cname,
6385
                code.error_goto_if_null(self.result(), self.pos)))
6386
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
6387

Robert Bradshaw's avatar
Robert Bradshaw committed
6388

6389
class PyCFunctionNode(ExprNode, ModuleNameMixin):
William Stein's avatar
William Stein committed
6390 6391 6392 6393
    #  Helper class used in the implementation of Python
    #  class definitions. Constructs a PyCFunction object
    #  from a PyMethodDef struct.
    #
6394
    #  pymethdef_cname   string             PyMethodDef structure
Robert Bradshaw's avatar
Robert Bradshaw committed
6395
    #  self_object       ExprNode or None
Robert Bradshaw's avatar
Robert Bradshaw committed
6396
    #  binding           bool
6397
    #  def_node          DefNode            the Python function node
6398
    #  module_name       EncodedString      Name of defining module
6399 6400
    #  code_object       CodeObjectNode     the PyCodeObject creator node

6401
    subexprs = ['code_object', 'defaults_tuple']
Stefan Behnel's avatar
Stefan Behnel committed
6402

Robert Bradshaw's avatar
Robert Bradshaw committed
6403
    self_object = None
6404
    code_object = None
Robert Bradshaw's avatar
Robert Bradshaw committed
6405
    binding = False
6406
    def_node = None
6407 6408 6409
    defaults = None
    defaults_struct = None
    defaults_pyobjects = 0
6410
    defaults_tuple = None
6411

6412 6413
    type = py_object_type
    is_temp = 1
6414

6415
    specialized_cpdefs = None
6416
    is_specialization = False
6417

6418 6419 6420 6421 6422 6423 6424 6425
    @classmethod
    def from_defnode(cls, node, binding):
        return cls(node.pos,
                   def_node=node,
                   pymethdef_cname=node.entry.pymethdef_cname,
                   binding=binding or node.specialized_cpdefs,
                   specialized_cpdefs=node.specialized_cpdefs,
                   code_object=CodeObjectNode(node))
6426

6427
    def analyse_types(self, env):
Robert Bradshaw's avatar
Robert Bradshaw committed
6428
        if self.binding:
6429
            if self.specialized_cpdefs or self.is_specialization:
Stefan Behnel's avatar
Stefan Behnel committed
6430 6431
                env.use_utility_code(
                    UtilityCode.load_cached("FusedFunction", "CythonFunction.c"))
6432
            else:
Stefan Behnel's avatar
Stefan Behnel committed
6433 6434
                env.use_utility_code(
                    UtilityCode.load_cached("CythonFunction", "CythonFunction.c"))
Mark Florisson's avatar
Mark Florisson committed
6435
            self.analyse_default_args(env)
6436

6437 6438 6439
        #TODO(craig,haoyu) This should be moved to a better place
        self.set_mod_name(env)

6440 6441 6442 6443 6444 6445
    def analyse_default_args(self, env):
        """
        Handle non-literal function's default arguments.
        """
        nonliteral_objects = []
        nonliteral_other = []
6446
        default_args = []
6447
        for arg in self.def_node.args:
6448 6449 6450 6451 6452 6453 6454
            if arg.default:
                if not arg.default.is_literal:
                    arg.is_dynamic = True
                    if arg.type.is_pyobject:
                        nonliteral_objects.append(arg)
                    else:
                        nonliteral_other.append(arg)
6455 6456
                else:
                    arg.default = DefaultLiteralArgNode(arg.pos, arg.default)
6457
                default_args.append(arg)
Mark Florisson's avatar
Mark Florisson committed
6458
        if nonliteral_objects or nonliteral_other:
6459 6460 6461 6462 6463 6464 6465 6466 6467 6468 6469 6470 6471 6472 6473 6474 6475 6476 6477 6478 6479 6480 6481
            module_scope = env.global_scope()
            cname = module_scope.next_id(Naming.defaults_struct_prefix)
            scope = Symtab.StructOrUnionScope(cname)
            self.defaults = []
            for arg in nonliteral_objects:
                entry = scope.declare_var(arg.name, arg.type, None,
                                          Naming.arg_prefix + arg.name,
                                          allow_pyobject=True)
                self.defaults.append((arg, entry))
            for arg in nonliteral_other:
                entry = scope.declare_var(arg.name, arg.type, None,
                                          Naming.arg_prefix + arg.name,
                                          allow_pyobject=False)
                self.defaults.append((arg, entry))
            entry = module_scope.declare_struct_or_union(
                None, 'struct', scope, 1, None, cname=cname)
            self.defaults_struct = scope
            self.defaults_pyobjects = len(nonliteral_objects)
            for arg, entry in self.defaults:
                arg.default_value = '%s->%s' % (
                    Naming.dynamic_args_cname, entry.cname)
            self.def_node.defaults_struct = self.defaults_struct.name

6482 6483 6484 6485 6486
        if default_args:
            if self.defaults_struct is None:
                self.defaults_tuple = TupleNode(self.pos, args=[
                    arg.default for arg in default_args])
                self.defaults_tuple.analyse_types(env)
6487 6488 6489 6490 6491 6492 6493 6494
            else:
                defaults_getter = Nodes.DefNode(
                    self.pos, args=[], star_arg=None, starstar_arg=None,
                    body=Nodes.ReturnStatNode(
                        self.pos, return_type=py_object_type,
                        value=DefaultsTupleNode(
                            self.pos, default_args,
                            self.defaults_struct)),
6495
                    decorators=None, name=StringEncoding.EncodedString("__defaults__"))
6496 6497 6498 6499 6500 6501 6502
                defaults_getter.analyse_declarations(env)
                defaults_getter.analyse_expressions(env)
                defaults_getter.body.analyse_expressions(
                    defaults_getter.local_scope)
                defaults_getter.py_wrapper_required = False
                defaults_getter.pymethdef_required = False
                self.def_node.defaults_getter = defaults_getter
6503

6504 6505
    def may_be_none(self):
        return False
6506

6507 6508
    gil_message = "Constructing Python function"

Stefan Behnel's avatar
Stefan Behnel committed
6509
    def self_result_code(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
6510 6511 6512 6513
        if self.self_object is None:
            self_result = "NULL"
        else:
            self_result = self.self_object.py_result()
Stefan Behnel's avatar
Stefan Behnel committed
6514 6515 6516
        return self_result

    def generate_result_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
6517
        if self.binding:
6518 6519 6520
            self.generate_cyfunction_code(code)
        else:
            self.generate_pycfunction_code(code)
6521

6522 6523 6524 6525 6526 6527 6528 6529 6530 6531 6532 6533 6534
    def generate_pycfunction_code(self, code):
        py_mod_name = self.get_py_mod_name(code)
        code.putln(
            '%s = PyCFunction_NewEx(&%s, %s, %s); %s' % (
                self.result(),
                self.pymethdef_cname,
                self.self_result_code(),
                py_mod_name,
                code.error_goto_if_null(self.result(), self.pos)))

        code.put_gotref(self.py_result())

    def generate_cyfunction_code(self, code):
6535 6536
        def_node = self.def_node

6537 6538
        if self.specialized_cpdefs:
            constructor = "__pyx_FusedFunction_NewEx"
6539
            def_node = self.specialized_cpdefs[0]
6540 6541
        elif self.is_specialization:
            constructor = "__pyx_FusedFunction_NewEx"
Robert Bradshaw's avatar
Robert Bradshaw committed
6542
        else:
6543 6544 6545 6546 6547 6548 6549 6550
            constructor = "__Pyx_CyFunction_NewEx"

        if self.code_object:
            code_object_result = self.code_object.py_result()
        else:
            code_object_result = 'NULL'

        flags = []
6551
        if def_node.is_staticmethod:
6552
            flags.append('__Pyx_CYFUNCTION_STATICMETHOD')
6553
        elif def_node.is_classmethod:
6554
            flags.append('__Pyx_CYFUNCTION_CLASSMETHOD')
6555 6556 6557 6558

        if def_node.local_scope.parent_scope.is_c_class_scope:
            flags.append('__Pyx_CYFUNCTION_CCLASS')

6559 6560 6561 6562
        if flags:
            flags = ' | '.join(flags)
        else:
            flags = '0'
6563

6564
        py_mod_name = self.get_py_mod_name(code)
William Stein's avatar
William Stein committed
6565
        code.putln(
6566
            '%s = %s(&%s, %s, %s, %s, %s); %s' % (
6567
                self.result(),
Robert Bradshaw's avatar
Robert Bradshaw committed
6568
                constructor,
William Stein's avatar
William Stein committed
6569
                self.pymethdef_cname,
6570
                flags,
Stefan Behnel's avatar
Stefan Behnel committed
6571
                self.self_result_code(),
6572
                py_mod_name,
6573
                code_object_result,
6574
                code.error_goto_if_null(self.result(), self.pos)))
6575

6576
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
6577

6578
        if def_node.requires_classobj:
6579 6580 6581 6582 6583 6584 6585 6586 6587
            assert code.pyclass_stack, "pyclass_stack is empty"
            class_node = code.pyclass_stack[-1]
            code.put_incref(self.py_result(), py_object_type)
            code.putln(
                'PyList_Append(%s, %s);' % (
                    class_node.class_cell.result(),
                    self.result()))
            code.put_giveref(self.py_result())

6588 6589 6590 6591 6592 6593 6594 6595 6596 6597 6598
        if self.defaults:
            code.putln(
                'if (!__Pyx_CyFunction_InitDefaults(%s, sizeof(%s), %d)) %s' % (
                    self.result(), self.defaults_struct.name,
                    self.defaults_pyobjects, code.error_goto(self.pos)))
            defaults = '__Pyx_CyFunction_Defaults(%s, %s)' % (
                self.defaults_struct.name, self.result())
            for arg, entry in self.defaults:
                arg.generate_assignment_code(code, target='%s->%s' % (
                    defaults, entry.cname))

6599 6600 6601
        if self.defaults_tuple:
            code.putln('__Pyx_CyFunction_SetDefaultsTuple(%s, %s);' % (
                self.result(), self.defaults_tuple.py_result()))
6602 6603 6604
        if def_node.defaults_getter:
            code.putln('__Pyx_CyFunction_SetDefaultsGetter(%s, %s);' % (
                self.result(), def_node.defaults_getter.entry.pyfunc_cname))
6605

6606

Stefan Behnel's avatar
Stefan Behnel committed
6607 6608 6609
class InnerFunctionNode(PyCFunctionNode):
    # Special PyCFunctionNode that depends on a closure class
    #
Vitja Makarov's avatar
Vitja Makarov committed
6610

Robert Bradshaw's avatar
Robert Bradshaw committed
6611
    binding = True
Vitja Makarov's avatar
Vitja Makarov committed
6612 6613
    needs_self_code = True

Stefan Behnel's avatar
Stefan Behnel committed
6614
    def self_result_code(self):
Vitja Makarov's avatar
Vitja Makarov committed
6615 6616 6617
        if self.needs_self_code:
            return "((PyObject*)%s)" % (Naming.cur_scope_cname)
        return "NULL"
Stefan Behnel's avatar
Stefan Behnel committed
6618

6619 6620 6621 6622
class CodeObjectNode(ExprNode):
    # Create a PyCodeObject for a CyFunction instance.
    #
    # def_node   DefNode    the Python function node
6623
    # varnames   TupleNode  a tuple with all local variable names
6624 6625 6626 6627 6628 6629 6630 6631 6632 6633 6634

    subexprs = ['varnames']
    is_temp = False

    def __init__(self, def_node):
        ExprNode.__init__(self, def_node.pos, def_node=def_node)
        args = list(def_node.args)
        if def_node.star_arg:
            args.append(def_node.star_arg)
        if def_node.starstar_arg:
            args.append(def_node.starstar_arg)
6635
        local_vars = [ arg for arg in def_node.local_scope.var_entries
6636
                       if arg.name ]
6637 6638
        self.varnames = TupleNode(
            def_node.pos,
6639
            args = [ IdentifierStringNode(arg.pos, value=arg.name)
6640
                     for arg in args + local_vars ],
6641 6642 6643 6644 6645 6646 6647 6648 6649 6650 6651 6652 6653 6654 6655 6656 6657 6658 6659 6660 6661 6662 6663 6664 6665
            is_temp = 0,
            is_literal = 1)

    def calculate_result_code(self):
        return self.result_code

    def generate_result_code(self, code):
        self.result_code = code.get_py_const(py_object_type, 'codeobj_', cleanup_level=2)

        code = code.get_cached_constants_writer()
        code.mark_pos(self.pos)
        func = self.def_node
        func_name = code.get_py_string_const(
            func.name, identifier=True, is_str=False, unicode_value=func.name)
        # FIXME: better way to get the module file path at module init time? Encoding to use?
        file_path = StringEncoding.BytesLiteral(func.pos[0].get_filenametable_entry().encode('utf8'))
        file_path_const = code.get_py_string_const(file_path, identifier=False, is_str=True)

        code.putln("%s = (PyObject*)__Pyx_PyCode_New(%d, %d, %d, 0, 0, %s, %s, %s, %s, %s, %s, %s, %s, %d, %s); %s" % (
            self.result_code,
            len(func.args),            # argcount
            func.num_kwonly_args,      # kwonlyargcount (Py3 only)
            len(self.varnames.args),   # nlocals
            Naming.empty_bytes,        # code
            Naming.empty_tuple,        # consts
6666 6667
            Naming.empty_tuple,        # names (FIXME)
            self.varnames.result(),    # varnames
6668 6669 6670 6671 6672 6673 6674 6675 6676 6677
            Naming.empty_tuple,        # freevars (FIXME)
            Naming.empty_tuple,        # cellvars (FIXME)
            file_path_const,           # filename
            func_name,                 # name
            self.pos[1],               # firstlineno
            Naming.empty_bytes,        # lnotab
            code.error_goto_if_null(self.result_code, self.pos),
            ))


6678 6679 6680 6681 6682 6683 6684 6685 6686 6687 6688 6689 6690 6691 6692 6693 6694 6695 6696 6697 6698 6699 6700 6701 6702 6703 6704 6705 6706 6707 6708
class DefaultLiteralArgNode(ExprNode):
    # CyFunction's literal argument default value
    #
    # Evaluate literal only once.

    subexprs = []
    is_literal = True
    is_temp = False

    def __init__(self, pos, arg):
        super(DefaultLiteralArgNode, self).__init__(pos)
        self.arg = arg
        self.type = self.arg.type
        self.evaluated = False

    def analyse_types(self, env):
        pass

    def generate_result_code(self, code):
        pass

    def generate_evaluation_code(self, code):
        if not self.evaluated:
            self.arg.generate_evaluation_code(code)
            self.evaluated = True

    def result(self):
        return self.type.cast_code(self.arg.result())


class DefaultNonLiteralArgNode(ExprNode):
6709 6710 6711 6712 6713
    # CyFunction's non-literal argument default value

    subexprs = []

    def __init__(self, pos, arg, defaults_struct):
6714
        super(DefaultNonLiteralArgNode, self).__init__(pos)
6715 6716 6717 6718 6719 6720 6721 6722 6723 6724 6725 6726 6727 6728 6729 6730 6731 6732 6733 6734 6735 6736 6737
        self.arg = arg
        self.defaults_struct = defaults_struct

    def analyse_types(self, env):
        self.type = self.arg.type
        self.is_temp = False

    def generate_result_code(self, code):
        pass

    def result(self):
        return '__Pyx_CyFunction_Defaults(%s, %s)->%s' % (
            self.defaults_struct.name, Naming.self_cname,
            self.defaults_struct.lookup(self.arg.name).cname)


class DefaultsTupleNode(TupleNode):
    # CyFunction's __defaults__ tuple

    def __init__(self, pos, defaults, defaults_struct):
        args = []
        for arg in defaults:
            if not arg.default.is_literal:
6738
                arg = DefaultNonLiteralArgNode(pos, arg, defaults_struct)
6739 6740 6741 6742 6743 6744
            else:
                arg = arg.default
            args.append(arg)
        super(DefaultsTupleNode, self).__init__(pos, args=args)


Stefan Behnel's avatar
Stefan Behnel committed
6745 6746 6747 6748 6749 6750 6751 6752 6753 6754 6755 6756 6757 6758 6759
class LambdaNode(InnerFunctionNode):
    # Lambda expression node (only used as a function reference)
    #
    # args          [CArgDeclNode]         formal arguments
    # star_arg      PyArgDeclNode or None  * argument
    # starstar_arg  PyArgDeclNode or None  ** argument
    # lambda_name   string                 a module-globally unique lambda name
    # result_expr   ExprNode
    # def_node      DefNode                the underlying function 'def' node

    child_attrs = ['def_node']

    name = StringEncoding.EncodedString('<lambda>')

    def analyse_declarations(self, env):
6760 6761
        self.def_node.no_assignment_synthesis = True
        self.def_node.pymethdef_required = True
Stefan Behnel's avatar
Stefan Behnel committed
6762
        self.def_node.analyse_declarations(env)
6763
        self.def_node.is_cyfunction = True
Stefan Behnel's avatar
Stefan Behnel committed
6764 6765 6766
        self.pymethdef_cname = self.def_node.entry.pymethdef_cname
        env.add_lambda_def(self.def_node)

6767 6768 6769 6770 6771 6772 6773 6774
    def analyse_types(self, env):
        self.def_node.analyse_expressions(env)
        super(LambdaNode, self).analyse_types(env)

    def generate_result_code(self, code):
        self.def_node.generate_execution_code(code)
        super(LambdaNode, self).generate_result_code(code)

6775

6776 6777 6778 6779 6780 6781 6782 6783
class GeneratorExpressionNode(LambdaNode):
    # A generator expression, e.g.  (i for i in range(10))
    #
    # Result is a generator.
    #
    # loop      ForStatNode   the for-loop, containing a YieldExprNode
    # def_node  DefNode       the underlying generator 'def' node

6784
    name = StringEncoding.EncodedString('genexpr')
6785 6786 6787
    binding = False

    def analyse_declarations(self, env):
6788 6789 6790
        super(GeneratorExpressionNode, self).analyse_declarations(env)
        # No pymethdef required
        self.def_node.pymethdef_required = False
6791
        self.def_node.py_wrapper_required = False
6792
        self.def_node.is_cyfunction = False
6793 6794
        # Force genexpr signature
        self.def_node.entry.signature = TypeSlots.pyfunction_noargs
6795 6796 6797

    def generate_result_code(self, code):
        code.putln(
6798
            '%s = %s(%s); %s' % (
6799
                self.result(),
6800
                self.def_node.entry.pyfunc_cname,
6801 6802 6803 6804 6805
                self.self_result_code(),
                code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.py_result())


6806 6807 6808 6809 6810
class YieldExprNode(ExprNode):
    # Yield expression node
    #
    # arg         ExprNode   the value to return from the generator
    # label_name  string     name of the C label used for this yield
6811
    # label_num   integer    yield label number
6812
    # is_yield_from  boolean is a YieldFromExprNode to delegate to another generator
6813 6814 6815

    subexprs = ['arg']
    type = py_object_type
6816
    label_num = 0
6817
    is_yield_from = False
6818 6819

    def analyse_types(self, env):
6820 6821
        if not self.label_num:
            error(self.pos, "'yield' not supported here")
6822 6823 6824 6825
        self.is_temp = 1
        if self.arg is not None:
            self.arg.analyse_types(env)
            if not self.arg.type.is_pyobject:
6826 6827 6828 6829
                self.coerce_yield_argument(env)

    def coerce_yield_argument(self, env):
        self.arg = self.arg.coerce_to_pyobject(env)
6830 6831 6832 6833 6834 6835 6836 6837 6838 6839 6840 6841 6842

    def generate_evaluation_code(self, code):
        if self.arg:
            self.arg.generate_evaluation_code(code)
            self.arg.make_owned_reference(code)
            code.putln(
                "%s = %s;" % (
                    Naming.retval_cname,
                    self.arg.result_as(py_object_type)))
            self.arg.generate_post_assignment_code(code)
            self.arg.free_temps(code)
        else:
            code.put_init_to_py_none(Naming.retval_cname, py_object_type)
6843 6844 6845 6846 6847 6848 6849 6850 6851 6852
        self.generate_yield_code(code)

    def generate_yield_code(self, code):
        """
        Generate the code to return the argument in 'Naming.retval_cname'
        and to continue at the yield label.
        """
        self.label_name = code.new_label('resume_from_yield')
        code.use_label(self.label_name)

6853
        saved = []
6854
        code.funcstate.closure_temps.reset()
6855
        for cname, type, manage_ref in code.funcstate.temps_in_use():
6856
            save_cname = code.funcstate.closure_temps.allocate_temp(type)
6857 6858 6859 6860
            saved.append((cname, save_cname, type))
            if type.is_pyobject:
                code.put_xgiveref(cname)
            code.putln('%s->%s = %s;' % (Naming.cur_scope_cname, save_cname, cname))
6861

6862
        code.put_xgiveref(Naming.retval_cname)
6863
        code.put_finish_refcount_context()
Stefan Behnel's avatar
Stefan Behnel committed
6864
        code.putln("/* return from generator, yielding value */")
6865 6866
        code.putln("%s->resume_label = %d;" % (
            Naming.generator_cname, self.label_num))
6867
        code.putln("return %s;" % Naming.retval_cname);
6868

6869
        code.put_label(self.label_name)
6870 6871 6872 6873 6874 6875
        for cname, save_cname, type in saved:
            code.putln('%s = %s->%s;' % (cname, Naming.cur_scope_cname, save_cname))
            if type.is_pyobject:
                code.putln('%s->%s = 0;' % (Naming.cur_scope_cname, save_cname))
            if type.is_pyobject:
                code.put_xgotref(cname)
6876 6877 6878 6879 6880 6881 6882 6883
        if self.result_is_used:
            self.allocate_temp_result(code)
            code.putln('%s = %s; %s' %
                       (self.result(), Naming.sent_value_cname,
                        code.error_goto_if_null(self.result(), self.pos)))
            code.put_incref(self.result(), py_object_type)
        else:
            code.putln(code.error_goto_if_null(Naming.sent_value_cname, self.pos))
6884

Vitja Makarov's avatar
Vitja Makarov committed
6885

6886 6887 6888 6889 6890 6891 6892 6893 6894 6895 6896 6897 6898 6899 6900 6901 6902 6903
class YieldFromExprNode(YieldExprNode):
    # "yield from GEN" expression
    is_yield_from = True

    def coerce_yield_argument(self, env):
        if not self.arg.type.is_string:
            # FIXME: support C arrays and C++ iterators?
            error(self.pos, "yielding from non-Python object not supported")
        self.arg = self.arg.coerce_to_pyobject(env)

    def generate_evaluation_code(self, code):
        code.globalstate.use_utility_code(UtilityCode.load_cached("YieldFrom", "Generator.c"))

        self.arg.generate_evaluation_code(code)
        code.putln("%s = __Pyx_Generator_Yield_From(%s, %s);" % (
            Naming.retval_cname,
            Naming.generator_cname,
            self.arg.result_as(py_object_type)))
6904
        self.arg.generate_disposal_code(code)
6905 6906 6907 6908 6909 6910 6911 6912 6913 6914 6915 6916 6917 6918 6919 6920 6921 6922 6923 6924 6925
        self.arg.free_temps(code)
        code.put_xgotref(Naming.retval_cname)

        code.putln("if (likely(%s)) {" % Naming.retval_cname)
        self.generate_yield_code(code)
        code.putln("} else {")
        # either error or sub-generator has normally terminated: return value => node result
        if self.result_is_used:
            # YieldExprNode has allocated the result temp for us
            code.putln("if (__Pyx_PyGen_FetchStopIterationValue(&%s) < 0) %s" % (
                self.result(),
                code.error_goto(self.pos)))
        else:
            code.putln("PyObject* exc_type = PyErr_Occurred();")
            code.putln("if (exc_type) {")
            code.putln("if (!PyErr_GivenExceptionMatches(exc_type, PyExc_StopIteration)) %s" %
                code.error_goto(self.pos))
            code.putln("PyErr_Clear();")
            code.putln("}")
        code.putln("}")

Vitja Makarov's avatar
Vitja Makarov committed
6926 6927 6928 6929 6930 6931 6932 6933 6934 6935 6936 6937 6938 6939 6940
class GlobalsExprNode(AtomicExprNode):
    type = dict_type
    is_temp = 1

    def analyse_types(self, env):
        env.use_utility_code(Builtin.globals_utility_code)

    gil_message = "Constructing globals dict"

    def generate_result_code(self, code):
        code.putln('%s = __Pyx_Globals(); %s' % (
            self.result(),
            code.error_goto_if_null(self.result(), self.pos)))
        code.put_gotref(self.result())

Vitja Makarov's avatar
Vitja Makarov committed
6941

6942 6943 6944 6945 6946 6947 6948 6949 6950 6951 6952
class LocalsDictItemNode(DictItemNode):
    def analyse_types(self, env):
        self.key.analyse_types(env)
        self.value.analyse_types(env)
        self.key = self.key.coerce_to_pyobject(env)
        if self.value.type.can_coerce_to_pyobject(env):
            self.value = self.value.coerce_to_pyobject(env)
        else:
            self.value = None


6953
class FuncLocalsExprNode(DictNode):
Vitja Makarov's avatar
Vitja Makarov committed
6954
    def __init__(self, pos, env):
6955
        local_vars = [entry.name for entry in env.entries.values()
6956 6957 6958 6959
                      if entry.name]
        items = [LocalsDictItemNode(
            pos, key=IdentifierStringNode(pos, value=var),
            value=NameNode(pos, name=var, allow_null=True))
Vitja Makarov's avatar
Vitja Makarov committed
6960 6961 6962 6963
                 for var in local_vars]
        DictNode.__init__(self, pos, key_value_pairs=items,
                          exclude_null_values=True)

6964 6965 6966 6967 6968
    def analyse_types(self, env):
        super(FuncLocalsExprNode, self).analyse_types(env)
        self.key_value_pairs = [i for i in self.key_value_pairs
                                if i.value is not None]

6969 6970 6971 6972 6973 6974 6975 6976 6977 6978 6979 6980 6981 6982 6983 6984 6985 6986 6987 6988 6989 6990 6991 6992 6993

class PyClassLocalsExprNode(AtomicExprNode):
    def __init__(self, pos, pyclass_dict):
        AtomicExprNode.__init__(self, pos)
        self.pyclass_dict = pyclass_dict

    def analyse_types(self, env):
        self.type = self.pyclass_dict.type
        self.is_tmep = 0

    def result(self):
        return self.pyclass_dict.result()

    def generate_result_code(self, code):
        pass


def LocalsExprNode(pos, scope_node, env):
    if env.is_module_scope:
        return GlobalsExprNode(pos)
    if env.is_py_class_scope:
        return PyClassLocalsExprNode(pos, scope_node.dict)
    return FuncLocalsExprNode(pos, env)


William Stein's avatar
William Stein committed
6994 6995 6996 6997 6998 6999
#-------------------------------------------------------------------
#
#  Unary operator nodes
#
#-------------------------------------------------------------------

7000 7001 7002 7003 7004 7005 7006
compile_time_unary_operators = {
    'not': operator.not_,
    '~': operator.inv,
    '-': operator.neg,
    '+': operator.pos,
}

7007
class UnopNode(ExprNode):
William Stein's avatar
William Stein committed
7008 7009 7010 7011 7012 7013 7014 7015 7016 7017
    #  operator     string
    #  operand      ExprNode
    #
    #  Processing during analyse_expressions phase:
    #
    #    analyse_c_operation
    #      Called when the operand is not a pyobject.
    #      - Check operand type and coerce if needed.
    #      - Determine result type and result code fragment.
    #      - Allocate temporary for result if needed.
7018

William Stein's avatar
William Stein committed
7019
    subexprs = ['operand']
Robert Bradshaw's avatar
Robert Bradshaw committed
7020
    infix = True
7021 7022 7023 7024

    def calculate_constant_result(self):
        func = compile_time_unary_operators[self.operator]
        self.constant_result = func(self.operand.constant_result)
7025

7026 7027 7028 7029 7030 7031 7032 7033 7034 7035 7036
    def compile_time_value(self, denv):
        func = compile_time_unary_operators.get(self.operator)
        if not func:
            error(self.pos,
                "Unary '%s' not supported in compile-time expression"
                    % self.operator)
        operand = self.operand.compile_time_value(denv)
        try:
            return func(operand)
        except Exception, e:
            self.compile_time_value_error(e)
7037

7038
    def infer_type(self, env):
7039
        operand_type = self.operand.infer_type(env)
7040 7041 7042 7043 7044 7045 7046
        if operand_type.is_cpp_class or operand_type.is_ptr:
            cpp_type = operand_type.find_cpp_operation_type(self.operator)
            if cpp_type is not None:
                return cpp_type
        return self.infer_unop_type(env, operand_type)

    def infer_unop_type(self, env, operand_type):
7047 7048 7049 7050
        if operand_type.is_pyobject:
            return py_object_type
        else:
            return operand_type
7051

William Stein's avatar
William Stein committed
7052 7053 7054 7055 7056 7057
    def analyse_types(self, env):
        self.operand.analyse_types(env)
        if self.is_py_operation():
            self.coerce_operand_to_pyobject(env)
            self.type = py_object_type
            self.is_temp = 1
7058 7059
        elif self.is_cpp_operation():
            self.analyse_cpp_operation(env)
William Stein's avatar
William Stein committed
7060 7061
        else:
            self.analyse_c_operation(env)
7062

William Stein's avatar
William Stein committed
7063
    def check_const(self):
7064
        return self.operand.check_const()
7065

William Stein's avatar
William Stein committed
7066 7067
    def is_py_operation(self):
        return self.operand.type.is_pyobject
7068

7069
    def nogil_check(self, env):
7070
        if self.is_py_operation():
7071
            self.gil_error()
7072

Danilo Freitas's avatar
Danilo Freitas committed
7073
    def is_cpp_operation(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
7074
        type = self.operand.type
Robert Bradshaw's avatar
Robert Bradshaw committed
7075
        return type.is_cpp_class
7076

William Stein's avatar
William Stein committed
7077 7078
    def coerce_operand_to_pyobject(self, env):
        self.operand = self.operand.coerce_to_pyobject(env)
7079

William Stein's avatar
William Stein committed
7080 7081 7082
    def generate_result_code(self, code):
        if self.operand.type.is_pyobject:
            self.generate_py_operation_code(code)
7083

William Stein's avatar
William Stein committed
7084 7085 7086
    def generate_py_operation_code(self, code):
        function = self.py_operation_function()
        code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
7087
            "%s = %s(%s); %s" % (
7088 7089
                self.result(),
                function,
William Stein's avatar
William Stein committed
7090
                self.operand.py_result(),
7091
                code.error_goto_if_null(self.result(), self.pos)))
7092
        code.put_gotref(self.py_result())
7093

William Stein's avatar
William Stein committed
7094 7095 7096 7097 7098 7099
    def type_error(self):
        if not self.operand.type.is_error:
            error(self.pos, "Invalid operand type for '%s' (%s)" %
                (self.operator, self.operand.type))
        self.type = PyrexTypes.error_type

Danilo Freitas's avatar
Danilo Freitas committed
7100
    def analyse_cpp_operation(self, env):
7101 7102 7103 7104
        cpp_type = self.operand.type.find_cpp_operation_type(self.operator)
        if cpp_type is None:
            error(self.pos, "'%s' operator not defined for %s" % (
                self.operator, type))
Danilo Freitas's avatar
Danilo Freitas committed
7105 7106
            self.type_error()
            return
7107
        self.type = cpp_type
Danilo Freitas's avatar
Danilo Freitas committed
7108

William Stein's avatar
William Stein committed
7109

7110
class NotNode(UnopNode):
William Stein's avatar
William Stein committed
7111 7112 7113
    #  'not' operator
    #
    #  operand   ExprNode
7114
    operator = '!'
7115

7116
    type = PyrexTypes.c_bint_type
7117 7118 7119 7120

    def calculate_constant_result(self):
        self.constant_result = not self.operand.constant_result

7121 7122 7123 7124 7125 7126 7127
    def compile_time_value(self, denv):
        operand = self.operand.compile_time_value(denv)
        try:
            return not operand
        except Exception, e:
            self.compile_time_value_error(e)

7128
    def infer_unop_type(self, env, operand_type):
7129
        return PyrexTypes.c_bint_type
7130

William Stein's avatar
William Stein committed
7131 7132
    def analyse_types(self, env):
        self.operand.analyse_types(env)
7133 7134 7135 7136 7137
        operand_type = self.operand.type
        if operand_type.is_cpp_class:
            cpp_type = operand_type.find_cpp_operation_type(self.operator)
            if not cpp_type:
                error(self.pos, "'!' operator not defined for %s" % operand_type)
7138 7139
                self.type = PyrexTypes.error_type
                return
7140
            self.type = cpp_type
7141 7142
        else:
            self.operand = self.operand.coerce_to_boolean(env)
7143

William Stein's avatar
William Stein committed
7144
    def calculate_result_code(self):
7145
        return "(!%s)" % self.operand.result()
7146

William Stein's avatar
William Stein committed
7147 7148 7149 7150 7151 7152
    def generate_result_code(self, code):
        pass


class UnaryPlusNode(UnopNode):
    #  unary '+' operator
7153

William Stein's avatar
William Stein committed
7154
    operator = '+'
7155

William Stein's avatar
William Stein committed
7156
    def analyse_c_operation(self, env):
Lisandro Dalcin's avatar
Lisandro Dalcin committed
7157
        self.type = PyrexTypes.widest_numeric_type(
Robert Bradshaw's avatar
Robert Bradshaw committed
7158
            self.operand.type, PyrexTypes.c_int_type)
7159

William Stein's avatar
William Stein committed
7160 7161
    def py_operation_function(self):
        return "PyNumber_Positive"
7162

William Stein's avatar
William Stein committed
7163
    def calculate_result_code(self):
7164 7165 7166 7167
        if self.is_cpp_operation():
            return "(+%s)" % self.operand.result()
        else:
            return self.operand.result()
William Stein's avatar
William Stein committed
7168 7169 7170 7171


class UnaryMinusNode(UnopNode):
    #  unary '-' operator
7172

William Stein's avatar
William Stein committed
7173
    operator = '-'
7174

William Stein's avatar
William Stein committed
7175 7176
    def analyse_c_operation(self, env):
        if self.operand.type.is_numeric:
7177 7178
            self.type = PyrexTypes.widest_numeric_type(
                self.operand.type, PyrexTypes.c_int_type)
7179 7180
        elif self.operand.type.is_enum:
            self.type = PyrexTypes.c_int_type
William Stein's avatar
William Stein committed
7181 7182
        else:
            self.type_error()
Robert Bradshaw's avatar
Robert Bradshaw committed
7183
        if self.type.is_complex:
7184
            self.infix = False
7185

William Stein's avatar
William Stein committed
7186 7187
    def py_operation_function(self):
        return "PyNumber_Negative"
7188

William Stein's avatar
William Stein committed
7189
    def calculate_result_code(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
7190 7191 7192 7193
        if self.infix:
            return "(-%s)" % self.operand.result()
        else:
            return "%s(%s)" % (self.operand.type.unary_op('-'), self.operand.result())
William Stein's avatar
William Stein committed
7194

7195 7196 7197 7198 7199
    def get_constant_c_result_code(self):
        value = self.operand.get_constant_c_result_code()
        if value:
            return "(-%s)" % (value)

William Stein's avatar
William Stein committed
7200 7201 7202 7203 7204
class TildeNode(UnopNode):
    #  unary '~' operator

    def analyse_c_operation(self, env):
        if self.operand.type.is_int:
7205 7206
            self.type = PyrexTypes.widest_numeric_type(
                self.operand.type, PyrexTypes.c_int_type)
7207 7208
        elif self.operand.type.is_enum:
            self.type = PyrexTypes.c_int_type
William Stein's avatar
William Stein committed
7209 7210 7211 7212 7213
        else:
            self.type_error()

    def py_operation_function(self):
        return "PyNumber_Invert"
7214

William Stein's avatar
William Stein committed
7215
    def calculate_result_code(self):
7216
        return "(~%s)" % self.operand.result()
William Stein's avatar
William Stein committed
7217 7218


7219 7220
class CUnopNode(UnopNode):

Robert Bradshaw's avatar
Robert Bradshaw committed
7221 7222 7223
    def is_py_operation(self):
        return False

7224 7225
class DereferenceNode(CUnopNode):
    #  unary * operator
7226 7227

    operator = '*'
7228

7229 7230 7231 7232 7233 7234
    def infer_unop_type(self, env, operand_type):
        if operand_type.is_ptr:
            return operand_type.base_type
        else:
            return PyrexTypes.error_type

Robert Bradshaw's avatar
Robert Bradshaw committed
7235 7236 7237 7238 7239 7240 7241 7242
    def analyse_c_operation(self, env):
        if self.operand.type.is_ptr:
            self.type = self.operand.type.base_type
        else:
            self.type_error()

    def calculate_result_code(self):
        return "(*%s)" % self.operand.result()
William Stein's avatar
William Stein committed
7243 7244


7245 7246
class DecrementIncrementNode(CUnopNode):
    #  unary ++/-- operator
7247

7248
    def analyse_c_operation(self, env):
7249 7250 7251 7252
        if self.operand.type.is_numeric:
            self.type = PyrexTypes.widest_numeric_type(
                self.operand.type, PyrexTypes.c_int_type)
        elif self.operand.type.is_ptr:
7253 7254 7255 7256 7257 7258 7259 7260 7261 7262 7263 7264 7265 7266
            self.type = self.operand.type
        else:
            self.type_error()

    def calculate_result_code(self):
        if self.is_prefix:
            return "(%s%s)" % (self.operator, self.operand.result())
        else:
            return "(%s%s)" % (self.operand.result(), self.operator)

def inc_dec_constructor(is_prefix, operator):
    return lambda pos, **kwds: DecrementIncrementNode(pos, is_prefix=is_prefix, operator=operator, **kwds)


7267
class AmpersandNode(CUnopNode):
William Stein's avatar
William Stein committed
7268 7269 7270
    #  The C address-of operator.
    #
    #  operand  ExprNode
7271
    operator = '&'
7272

7273 7274
    def infer_unop_type(self, env, operand_type):
        return PyrexTypes.c_ptr_type(operand_type)
William Stein's avatar
William Stein committed
7275 7276 7277 7278

    def analyse_types(self, env):
        self.operand.analyse_types(env)
        argtype = self.operand.type
7279 7280 7281 7282 7283
        if argtype.is_cpp_class:
            cpp_type = argtype.find_cpp_operation_type(self.operator)
            if cpp_type is not None:
                self.type = cpp_type
                return
7284
        if not (argtype.is_cfunction or argtype.is_reference or self.operand.is_addressable()):
7285 7286 7287 7288
            if argtype.is_memoryviewslice:
                self.error("Cannot take address of memoryview slice")
            else:
                self.error("Taking address of non-lvalue")
William Stein's avatar
William Stein committed
7289 7290 7291 7292 7293
            return
        if argtype.is_pyobject:
            self.error("Cannot take address of Python variable")
            return
        self.type = PyrexTypes.c_ptr_type(argtype)
7294

William Stein's avatar
William Stein committed
7295
    def check_const(self):
7296
        return self.operand.check_const_addr()
7297

William Stein's avatar
William Stein committed
7298 7299 7300 7301
    def error(self, mess):
        error(self.pos, mess)
        self.type = PyrexTypes.error_type
        self.result_code = "<error>"
7302

William Stein's avatar
William Stein committed
7303
    def calculate_result_code(self):
7304
        return "(&%s)" % self.operand.result()
William Stein's avatar
William Stein committed
7305 7306 7307

    def generate_result_code(self, code):
        pass
7308

William Stein's avatar
William Stein committed
7309 7310 7311 7312 7313 7314 7315 7316

unop_node_classes = {
    "+":  UnaryPlusNode,
    "-":  UnaryMinusNode,
    "~":  TildeNode,
}

def unop_node(pos, operator, operand):
7317
    # Construct unnop node of appropriate class for
William Stein's avatar
William Stein committed
7318
    # given operator.
7319
    if isinstance(operand, IntNode) and operator == '-':
7320
        return IntNode(pos = operand.pos, value = str(-Utils.str_to_number(operand.value)))
7321 7322
    elif isinstance(operand, UnopNode) and operand.operator == operator in '+-':
        warning(pos, "Python has no increment/decrement operator: %s%sx == %s(%sx) == x" % ((operator,)*4), 5)
7323 7324
    return unop_node_classes[operator](pos,
        operator = operator,
William Stein's avatar
William Stein committed
7325 7326 7327
        operand = operand)


7328
class TypecastNode(ExprNode):
William Stein's avatar
William Stein committed
7329 7330
    #  C type cast
    #
7331
    #  operand      ExprNode
William Stein's avatar
William Stein committed
7332 7333
    #  base_type    CBaseTypeNode
    #  declarator   CDeclaratorNode
7334 7335 7336
    #
    #  If used from a transform, one can if wanted specify the attribute
    #  "type" directly and leave base_type and declarator to None
7337

William Stein's avatar
William Stein committed
7338
    subexprs = ['operand']
7339
    base_type = declarator = type = None
7340

Robert Bradshaw's avatar
Robert Bradshaw committed
7341
    def type_dependencies(self, env):
7342
        return ()
7343

Robert Bradshaw's avatar
Robert Bradshaw committed
7344
    def infer_type(self, env):
7345 7346 7347 7348
        if self.type is None:
            base_type = self.base_type.analyse(env)
            _, self.type = self.declarator.analyse(base_type, env)
        return self.type
7349

William Stein's avatar
William Stein committed
7350
    def analyse_types(self, env):
7351 7352 7353
        if self.type is None:
            base_type = self.base_type.analyse(env)
            _, self.type = self.declarator.analyse(base_type, env)
Robert Bradshaw's avatar
Robert Bradshaw committed
7354 7355 7356
        if self.operand.has_constant_result():
            # Must be done after self.type is resolved.
            self.calculate_constant_result()
7357 7358 7359 7360
        if self.type.is_cfunction:
            error(self.pos,
                "Cannot cast to a function type")
            self.type = PyrexTypes.error_type
William Stein's avatar
William Stein committed
7361 7362 7363
        self.operand.analyse_types(env)
        to_py = self.type.is_pyobject
        from_py = self.operand.type.is_pyobject
7364 7365 7366
        if from_py and not to_py and self.operand.is_ephemeral():
            if not self.type.is_numeric and not self.type.is_cpp_class:
                error(self.pos, "Casting temporary Python object to non-numeric non-Python type")
William Stein's avatar
William Stein committed
7367
        if to_py and not from_py:
7368 7369 7370 7371 7372 7373
            if self.type is bytes_type and self.operand.type.is_int:
                # FIXME: the type cast node isn't needed in this case
                # and can be dropped once analyse_types() can return a
                # different node
                self.operand = CoerceIntToBytesNode(self.operand, env)
            elif self.operand.type.can_coerce_to_pyobject(env):
Dag Sverre Seljebotn's avatar
Merge  
Dag Sverre Seljebotn committed
7374
                self.result_ctype = py_object_type
7375
                self.operand = self.operand.coerce_to_pyobject(env)
7376
            else:
7377 7378 7379 7380
                if self.operand.type.is_ptr:
                    if not (self.operand.type.base_type.is_void or self.operand.type.base_type.is_struct):
                        error(self.pos, "Python objects cannot be cast from pointers of primitive types")
                else:
7381
                    # Should this be an error?
7382
                    warning(self.pos, "No conversion from %s to %s, python object pointer used." % (self.operand.type, self.type))
7383
                self.operand = self.operand.coerce_to_simple(env)
7384
        elif from_py and not to_py:
7385
            if self.type.create_from_py_utility_code(env):
7386
                self.operand = self.operand.coerce_to(self.type, env)
7387 7388 7389
            elif self.type.is_ptr:
                if not (self.type.base_type.is_void or self.type.base_type.is_struct):
                    error(self.pos, "Python objects cannot be cast to pointers of primitive types")
7390 7391
            else:
                warning(self.pos, "No conversion from %s to %s, python object pointer used." % (self.type, self.operand.type))
7392 7393
        elif from_py and to_py:
            if self.typecheck and self.type.is_extension_type:
7394
                self.operand = PyTypeTestNode(self.operand, self.type, env, notnone=True)
7395 7396
        elif self.type.is_complex and self.operand.type.is_complex:
            self.operand = self.operand.coerce_to_simple(env)
7397 7398
        elif self.operand.type.is_fused:
            self.operand = self.operand.coerce_to(self.type, env)
7399
            #self.type = self.operand.type
7400

Stefan Behnel's avatar
Stefan Behnel committed
7401
    def is_simple(self):
7402 7403
        # either temp or a C cast => no side effects other than the operand's
        return self.operand.is_simple()
Stefan Behnel's avatar
Stefan Behnel committed
7404

7405
    def nonlocally_immutable(self):
7406
        return self.is_temp or self.operand.nonlocally_immutable()
7407

7408 7409 7410
    def nogil_check(self, env):
        if self.type and self.type.is_pyobject and self.is_temp:
            self.gil_error()
7411

William Stein's avatar
William Stein committed
7412
    def check_const(self):
7413
        return self.operand.check_const()
Stefan Behnel's avatar
Stefan Behnel committed
7414 7415

    def calculate_constant_result(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
7416
        self.constant_result = self.calculate_result_code(self.operand.constant_result)
7417

Robert Bradshaw's avatar
Robert Bradshaw committed
7418 7419 7420
    def calculate_result_code(self, operand_result = None):
        if operand_result is None:
            operand_result = self.operand.result()
7421 7422 7423 7424 7425 7426 7427 7428 7429 7430 7431
        if self.type.is_complex:
            operand_result = self.operand.result()
            if self.operand.type.is_complex:
                real_part = self.type.real_type.cast_code("__Pyx_CREAL(%s)" % operand_result)
                imag_part = self.type.real_type.cast_code("__Pyx_CIMAG(%s)" % operand_result)
            else:
                real_part = self.type.real_type.cast_code(operand_result)
                imag_part = "0"
            return "%s(%s, %s)" % (
                    self.type.from_parts,
                    real_part,
7432
                    imag_part)
7433
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
7434
            return self.type.cast_code(operand_result)
7435

7436 7437 7438 7439
    def get_constant_c_result_code(self):
        operand_result = self.operand.get_constant_c_result_code()
        if operand_result:
            return self.type.cast_code(operand_result)
7440

William Stein's avatar
William Stein committed
7441 7442 7443 7444 7445 7446 7447 7448 7449 7450 7451
    def result_as(self, type):
        if self.type.is_pyobject and not self.is_temp:
            #  Optimise away some unnecessary casting
            return self.operand.result_as(type)
        else:
            return ExprNode.result_as(self, type)

    def generate_result_code(self, code):
        if self.is_temp:
            code.putln(
                "%s = (PyObject *)%s;" % (
7452 7453 7454
                    self.result(),
                    self.operand.result()))
            code.put_incref(self.result(), self.ctype())
William Stein's avatar
William Stein committed
7455 7456


7457 7458 7459 7460
ERR_START = "Start may not be given"
ERR_NOT_STOP = "Stop must be provided to indicate shape"
ERR_STEPS = ("Strides may only be given to indicate contiguity. "
             "Consider slicing it after conversion")
7461
ERR_NOT_POINTER = "Can only create cython.array from pointer or array"
7462 7463 7464 7465 7466 7467 7468
ERR_BASE_TYPE = "Pointer base type does not match cython.array base type"

class CythonArrayNode(ExprNode):
    """
    Used when a pointer of base_type is cast to a memoryviewslice with that
    base type. i.e.

7469
        <int[:M:1, :N]> p
7470 7471 7472 7473 7474 7475 7476

    creates a fortran-contiguous cython.array.

    We leave the type set to object so coercions to object are more efficient
    and less work. Acquiring a memoryviewslice from this will be just as
    efficient. ExprNode.coerce_to() will do the additional typecheck on
    self.compile_time_type
7477 7478 7479 7480 7481 7482

    This also handles <int[:, :]> my_c_array


    operand             ExprNode                 the thing we're casting
    base_type_node      MemoryViewSliceTypeNode  the cast expression node
7483 7484 7485 7486 7487 7488 7489
    """

    subexprs = ['operand', 'shapes']

    shapes = None
    is_temp = True
    mode = "c"
7490
    array_dtype = None
7491 7492 7493 7494 7495 7496

    shape_type = PyrexTypes.c_py_ssize_t_type

    def analyse_types(self, env):
        import MemoryView

7497 7498 7499 7500 7501 7502 7503 7504 7505
        self.operand.analyse_types(env)
        if self.array_dtype:
            array_dtype = self.array_dtype
        else:
            array_dtype = self.base_type_node.base_type_node.analyse(env)
        axes = self.base_type_node.axes

        MemoryView.validate_memslice_dtype(self.pos, array_dtype)

7506 7507
        self.type = error_type
        self.shapes = []
7508
        ndim = len(axes)
7509

7510 7511 7512
        # Base type of the pointer or C array we are converting
        base_type = self.operand.type

7513 7514 7515
        if not self.operand.type.is_ptr and not self.operand.type.is_array:
            return error(self.operand.pos, ERR_NOT_POINTER)

7516 7517 7518 7519 7520 7521
        # Dimension sizes of C array
        array_dimension_sizes = []
        if base_type.is_array:
            while base_type.is_array:
                array_dimension_sizes.append(base_type.size)
                base_type = base_type.base_type
7522
        elif base_type.is_ptr:
7523
            base_type = base_type.base_type
7524 7525
        else:
            return error()
7526

7527
        if not (base_type.same_as(array_dtype) or base_type.is_void):
7528 7529 7530 7531 7532 7533 7534 7535 7536 7537
            return error(self.operand.pos, ERR_BASE_TYPE)
        elif self.operand.type.is_array and len(array_dimension_sizes) != ndim:
            return error(self.operand.pos,
                         "Expected %d dimensions, array has %d dimensions" %
                                            (ndim, len(array_dimension_sizes)))

        # Verify the start, stop and step values
        # In case of a C array, use the size of C array in each dimension to
        # get an automatic cast
        for axis_no, axis in enumerate(axes):
7538 7539 7540 7541
            if not axis.start.is_none:
                return error(axis.start.pos, ERR_START)

            if axis.stop.is_none:
7542 7543 7544 7545 7546 7547 7548
                if array_dimension_sizes:
                    dimsize = array_dimension_sizes[axis_no]
                    axis.stop = IntNode(self.pos, value=dimsize,
                                        constant_result=dimsize,
                                        type=PyrexTypes.c_int_type)
                else:
                    return error(axis.pos, ERR_NOT_STOP)
7549 7550 7551 7552 7553 7554 7555 7556

            axis.stop.analyse_types(env)
            shape = axis.stop.coerce_to(self.shape_type, env)
            if not shape.is_literal:
                shape.coerce_to_temp(env)

            self.shapes.append(shape)

7557
            first_or_last = axis_no in (0, ndim - 1)
7558
            if not axis.step.is_none and first_or_last:
7559
                # '1' in the first or last dimension denotes F or C contiguity
7560 7561 7562 7563 7564 7565 7566 7567 7568 7569 7570
                axis.step.analyse_types(env)
                if (not axis.step.type.is_int and axis.step.is_literal and not
                        axis.step.type.is_error):
                    return error(axis.step.pos, "Expected an integer literal")

                if axis.step.compile_time_value(env) != 1:
                    return error(axis.step.pos, ERR_STEPS)

                if axis_no == 0:
                    self.mode = "fortran"

7571 7572
            elif not axis.step.is_none and not first_or_last:
                # step provided in some other dimension
7573 7574 7575 7576 7577
                return error(axis.step.pos, ERR_STEPS)

        if not self.operand.is_name:
            self.operand = self.operand.coerce_to_temp(env)

7578
        axes = [('direct', 'follow')] * len(axes)
7579 7580 7581 7582 7583 7584
        if self.mode == "fortran":
            axes[0] = ('direct', 'contig')
        else:
            axes[-1] = ('direct', 'contig')

        self.coercion_type = PyrexTypes.MemoryViewSliceType(array_dtype, axes)
7585
        self.type = self.get_cython_array_type(env)
7586
        MemoryView.use_cython_array_utility_code(env)
7587 7588 7589 7590 7591 7592 7593 7594
        env.use_utility_code(MemoryView.typeinfo_to_format_code)

    def allocate_temp_result(self, code):
        if self.temp_code:
            raise RuntimeError("temp allocated mulitple times")

        self.temp_code = code.funcstate.allocate_temp(self.type, True)

7595 7596 7597 7598
    def infer_type(self, env):
        return self.get_cython_array_type(env)

    def get_cython_array_type(self, env):
7599
        return env.global_scope().context.cython_scope.viewscope.lookup("array").type
7600

7601 7602 7603 7604 7605 7606 7607 7608 7609 7610 7611 7612 7613
    def generate_result_code(self, code):
        import Buffer

        shapes = [self.shape_type.cast_code(shape.result())
                      for shape in self.shapes]
        dtype = self.coercion_type.dtype

        shapes_temp = code.funcstate.allocate_temp(py_object_type, True)
        format_temp = code.funcstate.allocate_temp(py_object_type, True)

        itemsize = "sizeof(%s)" % dtype.declaration_code("")
        type_info = Buffer.get_type_information_cname(code, dtype)

7614 7615 7616 7617 7618 7619
        if self.operand.type.is_ptr:
            code.putln("if (!%s) {" % self.operand.result())
            code.putln(    'PyErr_SetString(PyExc_ValueError,'
                                '"Cannot create cython.array from NULL pointer");')
            code.putln(code.error_goto(self.operand.pos))
            code.putln("}")
7620 7621 7622

        code.putln("%s = __pyx_format_from_typeinfo(&%s);" %
                                                (format_temp, type_info))
7623 7624 7625 7626
        buildvalue_fmt = " __PYX_BUILD_PY_SSIZE_T " * len(shapes)
        code.putln('%s = Py_BuildValue("(" %s ")", %s);' % (shapes_temp,
                                                            buildvalue_fmt,
                                                            ", ".join(shapes)))
7627

7628 7629 7630
        err = "!%s || !%s || !PyBytes_AsString(%s)" % (format_temp,
                                                       shapes_temp,
                                                       format_temp)
7631 7632 7633 7634 7635 7636 7637 7638
        code.putln(code.error_goto_if(err, self.pos))
        code.put_gotref(format_temp)
        code.put_gotref(shapes_temp)

        tup = (self.result(), shapes_temp, itemsize, format_temp,
               self.mode, self.operand.result())
        code.putln('%s = __pyx_array_new('
                            '%s, %s, PyBytes_AS_STRING(%s), '
7639
                            '(char *) "%s", (char *) %s);' % tup)
7640 7641 7642 7643 7644 7645 7646 7647 7648 7649
        code.putln(code.error_goto_if_null(self.result(), self.pos))
        code.put_gotref(self.result())

        def dispose(temp):
            code.put_decref_clear(temp, py_object_type)
            code.funcstate.release_temp(temp)

        dispose(shapes_temp)
        dispose(format_temp)

7650 7651 7652 7653 7654 7655 7656 7657 7658 7659 7660 7661 7662 7663 7664 7665 7666 7667 7668 7669 7670 7671 7672
    @classmethod
    def from_carray(cls, src_node, env):
        """
        Given a C array type, return a CythonArrayNode
        """
        pos = src_node.pos
        base_type = src_node.type

        none_node = NoneNode(pos)
        axes = []

        while base_type.is_array:
            axes.append(SliceNode(pos, start=none_node, stop=none_node,
                                       step=none_node))
            base_type = base_type.base_type
        axes[-1].step = IntNode(pos, value="1", is_c_literal=True)

        memslicenode = Nodes.MemoryViewSliceTypeNode(pos, axes=axes,
                                                     base_type_node=base_type)
        result = CythonArrayNode(pos, base_type_node=memslicenode,
                                 operand=src_node, array_dtype=base_type)
        result.analyse_types(env)
        return result
7673

7674
class SizeofNode(ExprNode):
William Stein's avatar
William Stein committed
7675
    #  Abstract base class for sizeof(x) expression nodes.
7676

7677
    type = PyrexTypes.c_size_t_type
William Stein's avatar
William Stein committed
7678 7679

    def check_const(self):
7680
        return True
William Stein's avatar
William Stein committed
7681 7682 7683 7684 7685 7686 7687 7688 7689 7690

    def generate_result_code(self, code):
        pass


class SizeofTypeNode(SizeofNode):
    #  C sizeof function applied to a type
    #
    #  base_type   CBaseTypeNode
    #  declarator  CDeclaratorNode
7691

William Stein's avatar
William Stein committed
7692
    subexprs = []
7693
    arg_type = None
7694

William Stein's avatar
William Stein committed
7695
    def analyse_types(self, env):
7696 7697
        # we may have incorrectly interpreted a dotted name as a type rather than an attribute
        # this could be better handled by more uniformly treating types as runtime-available objects
7698
        if 0 and self.base_type.module_path:
7699 7700 7701 7702 7703 7704 7705 7706 7707 7708 7709
            path = self.base_type.module_path
            obj = env.lookup(path[0])
            if obj.as_module is None:
                operand = NameNode(pos=self.pos, name=path[0])
                for attr in path[1:]:
                    operand = AttributeNode(pos=self.pos, obj=operand, attribute=attr)
                operand = AttributeNode(pos=self.pos, obj=operand, attribute=self.base_type.name)
                self.operand = operand
                self.__class__ = SizeofVarNode
                self.analyse_types(env)
                return
7710 7711 7712 7713
        if self.arg_type is None:
            base_type = self.base_type.analyse(env)
            _, arg_type = self.declarator.analyse(base_type, env)
            self.arg_type = arg_type
7714
        self.check_type()
7715

7716 7717
    def check_type(self):
        arg_type = self.arg_type
7718
        if arg_type.is_pyobject and not arg_type.is_extension_type:
William Stein's avatar
William Stein committed
7719 7720 7721 7722 7723
            error(self.pos, "Cannot take sizeof Python object")
        elif arg_type.is_void:
            error(self.pos, "Cannot take sizeof void")
        elif not arg_type.is_complete():
            error(self.pos, "Cannot take sizeof incomplete type '%s'" % arg_type)
7724

William Stein's avatar
William Stein committed
7725
    def calculate_result_code(self):
7726 7727 7728 7729 7730 7731
        if self.arg_type.is_extension_type:
            # the size of the pointer is boring
            # we want the size of the actual struct
            arg_code = self.arg_type.declaration_code("", deref=1)
        else:
            arg_code = self.arg_type.declaration_code("")
William Stein's avatar
William Stein committed
7732
        return "(sizeof(%s))" % arg_code
7733

William Stein's avatar
William Stein committed
7734 7735 7736 7737 7738

class SizeofVarNode(SizeofNode):
    #  C sizeof function applied to a variable
    #
    #  operand   ExprNode
7739

William Stein's avatar
William Stein committed
7740
    subexprs = ['operand']
7741

William Stein's avatar
William Stein committed
7742
    def analyse_types(self, env):
7743 7744 7745 7746 7747
        # We may actually be looking at a type rather than a variable...
        # If we are, traditional analysis would fail...
        operand_as_type = self.operand.analyse_as_type(env)
        if operand_as_type:
            self.arg_type = operand_as_type
Mark Florisson's avatar
Mark Florisson committed
7748 7749
            if self.arg_type.is_fused:
                self.arg_type = self.arg_type.specialize(env.fused_to_specific)
7750 7751 7752 7753
            self.__class__ = SizeofTypeNode
            self.check_type()
        else:
            self.operand.analyse_types(env)
7754

William Stein's avatar
William Stein committed
7755
    def calculate_result_code(self):
7756
        return "(sizeof(%s))" % self.operand.result()
7757

William Stein's avatar
William Stein committed
7758 7759 7760
    def generate_result_code(self, code):
        pass

Robert Bradshaw's avatar
Robert Bradshaw committed
7761
class TypeofNode(ExprNode):
7762 7763 7764
    #  Compile-time type of an expression, as a string.
    #
    #  operand   ExprNode
Robert Bradshaw's avatar
Robert Bradshaw committed
7765
    #  literal   StringNode # internal
7766

Robert Bradshaw's avatar
Robert Bradshaw committed
7767 7768
    literal = None
    type = py_object_type
7769

Stefan Behnel's avatar
Stefan Behnel committed
7770
    subexprs = ['literal'] # 'operand' will be ignored after type analysis!
7771

7772 7773
    def analyse_types(self, env):
        self.operand.analyse_types(env)
7774
        value = StringEncoding.EncodedString(str(self.operand.type)) #self.operand.type.typeof_name())
7775
        self.literal = StringNode(self.pos, value=value)
Robert Bradshaw's avatar
Robert Bradshaw committed
7776 7777
        self.literal.analyse_types(env)
        self.literal = self.literal.coerce_to_pyobject(env)
7778 7779 7780 7781

    def may_be_none(self):
        return False

7782
    def generate_evaluation_code(self, code):
Robert Bradshaw's avatar
Robert Bradshaw committed
7783
        self.literal.generate_evaluation_code(code)
7784

Robert Bradshaw's avatar
Robert Bradshaw committed
7785 7786
    def calculate_result_code(self):
        return self.literal.calculate_result_code()
William Stein's avatar
William Stein committed
7787 7788 7789 7790 7791 7792 7793

#-------------------------------------------------------------------
#
#  Binary operator nodes
#
#-------------------------------------------------------------------

Stefan Behnel's avatar
Stefan Behnel committed
7794 7795 7796
def _not_in(x, seq):
    return x not in seq

7797 7798 7799
compile_time_binary_operators = {
    '<': operator.lt,
    '<=': operator.le,
7800
    '==': operator.eq,
7801 7802 7803 7804 7805 7806 7807
    '!=': operator.ne,
    '>=': operator.ge,
    '>': operator.gt,
    'is': operator.is_,
    'is_not': operator.is_not,
    '+': operator.add,
    '&': operator.and_,
7808
    '/': operator.truediv,
7809 7810 7811 7812 7813 7814 7815 7816 7817
    '//': operator.floordiv,
    '<<': operator.lshift,
    '%': operator.mod,
    '*': operator.mul,
    '|': operator.or_,
    '**': operator.pow,
    '>>': operator.rshift,
    '-': operator.sub,
    '^': operator.xor,
Stefan Behnel's avatar
Stefan Behnel committed
7818 7819
    'in': operator.contains,
    'not_in': _not_in,
7820 7821 7822 7823 7824 7825 7826
}

def get_compile_time_binop(node):
    func = compile_time_binary_operators.get(node.operator)
    if not func:
        error(node.pos,
            "Binary '%s' not supported in compile-time expression"
7827
                % node.operator)
7828 7829
    return func

7830
class BinopNode(ExprNode):
William Stein's avatar
William Stein committed
7831 7832 7833 7834 7835 7836 7837 7838 7839 7840 7841
    #  operator     string
    #  operand1     ExprNode
    #  operand2     ExprNode
    #
    #  Processing during analyse_expressions phase:
    #
    #    analyse_c_operation
    #      Called when neither operand is a pyobject.
    #      - Check operand types and coerce if needed.
    #      - Determine result type and result code fragment.
    #      - Allocate temporary for result if needed.
7842

William Stein's avatar
William Stein committed
7843
    subexprs = ['operand1', 'operand2']
7844
    inplace = False
7845 7846 7847 7848 7849 7850 7851

    def calculate_constant_result(self):
        func = compile_time_binary_operators[self.operator]
        self.constant_result = func(
            self.operand1.constant_result,
            self.operand2.constant_result)

7852 7853 7854 7855 7856 7857 7858 7859
    def compile_time_value(self, denv):
        func = get_compile_time_binop(self)
        operand1 = self.operand1.compile_time_value(denv)
        operand2 = self.operand2.compile_time_value(denv)
        try:
            return func(operand1, operand2)
        except Exception, e:
            self.compile_time_value_error(e)
7860

7861 7862
    def infer_type(self, env):
        return self.result_type(self.operand1.infer_type(env),
Robert Bradshaw's avatar
Robert Bradshaw committed
7863
                                self.operand2.infer_type(env))
7864

William Stein's avatar
William Stein committed
7865 7866 7867
    def analyse_types(self, env):
        self.operand1.analyse_types(env)
        self.operand2.analyse_types(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
7868
        self.analyse_operation(env)
7869

Robert Bradshaw's avatar
Robert Bradshaw committed
7870
    def analyse_operation(self, env):
William Stein's avatar
William Stein committed
7871 7872
        if self.is_py_operation():
            self.coerce_operands_to_pyobjects(env)
7873 7874 7875
            self.type = self.result_type(self.operand1.type,
                                         self.operand2.type)
            assert self.type.is_pyobject
William Stein's avatar
William Stein committed
7876
            self.is_temp = 1
DaniloFreitas's avatar
DaniloFreitas committed
7877 7878
        elif self.is_cpp_operation():
            self.analyse_cpp_operation(env)
William Stein's avatar
William Stein committed
7879 7880
        else:
            self.analyse_c_operation(env)
7881

William Stein's avatar
William Stein committed
7882
    def is_py_operation(self):
7883
        return self.is_py_operation_types(self.operand1.type, self.operand2.type)
7884

7885 7886 7887
    def is_py_operation_types(self, type1, type2):
        return type1.is_pyobject or type2.is_pyobject

DaniloFreitas's avatar
DaniloFreitas committed
7888
    def is_cpp_operation(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
7889 7890
        return (self.operand1.type.is_cpp_class
            or self.operand2.type.is_cpp_class)
7891

7892 7893 7894 7895 7896 7897 7898 7899 7900 7901 7902 7903 7904 7905 7906 7907
    def analyse_cpp_operation(self, env):
        type1 = self.operand1.type
        type2 = self.operand2.type
        entry = env.lookup_operator(self.operator, [self.operand1, self.operand2])
        if not entry:
            self.type_error()
            return
        func_type = entry.type
        if func_type.is_ptr:
            func_type = func_type.base_type
        if len(func_type.args) == 1:
            self.operand2 = self.operand2.coerce_to(func_type.args[0].type, env)
        else:
            self.operand1 = self.operand1.coerce_to(func_type.args[0].type, env)
            self.operand2 = self.operand2.coerce_to(func_type.args[1].type, env)
        self.type = func_type.return_type
7908

7909 7910
    def result_type(self, type1, type2):
        if self.is_py_operation_types(type1, type2):
7911 7912 7913 7914 7915 7916 7917 7918 7919 7920 7921 7922 7923 7924 7925 7926 7927 7928
            if type2.is_string:
                type2 = Builtin.bytes_type
            if type1.is_string:
                type1 = Builtin.bytes_type
            elif self.operator == '%' \
                     and type1 in (Builtin.str_type, Builtin.unicode_type):
                # note that  b'%s' % b'abc'  doesn't work in Py3
                return type1
            if type1.is_builtin_type:
                if type1 is type2:
                    if self.operator in '**%+|&^':
                        # FIXME: at least these operators should be safe - others?
                        return type1
                elif self.operator == '*':
                    if type1 in (Builtin.bytes_type, Builtin.str_type, Builtin.unicode_type):
                        return type1
                    # multiplication of containers/numbers with an
                    # integer value always (?) returns the same type
7929
                    if type2.is_int:
7930
                        return type1
7931 7932 7933 7934
            elif type2.is_builtin_type and type1.is_int and self.operator == '*':
                # multiplication of containers/numbers with an
                # integer value always (?) returns the same type
                return type2
7935 7936 7937
            return py_object_type
        else:
            return self.compute_c_result_type(type1, type2)
7938

7939
    def nogil_check(self, env):
7940
        if self.is_py_operation():
7941
            self.gil_error()
7942

William Stein's avatar
William Stein committed
7943 7944 7945
    def coerce_operands_to_pyobjects(self, env):
        self.operand1 = self.operand1.coerce_to_pyobject(env)
        self.operand2 = self.operand2.coerce_to_pyobject(env)
7946

William Stein's avatar
William Stein committed
7947
    def check_const(self):
7948
        return self.operand1.check_const() and self.operand2.check_const()
7949

William Stein's avatar
William Stein committed
7950 7951 7952 7953
    def generate_result_code(self, code):
        #print "BinopNode.generate_result_code:", self.operand1, self.operand2 ###
        if self.operand1.type.is_pyobject:
            function = self.py_operation_function()
7954
            if self.operator == '**':
William Stein's avatar
William Stein committed
7955 7956 7957 7958
                extra_args = ", Py_None"
            else:
                extra_args = ""
            code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
7959
                "%s = %s(%s, %s%s); %s" % (
7960 7961
                    self.result(),
                    function,
William Stein's avatar
William Stein committed
7962 7963 7964
                    self.operand1.py_result(),
                    self.operand2.py_result(),
                    extra_args,
7965
                    code.error_goto_if_null(self.result(), self.pos)))
7966
            code.put_gotref(self.py_result())
7967

William Stein's avatar
William Stein committed
7968 7969 7970 7971
    def type_error(self):
        if not (self.operand1.type.is_error
                or self.operand2.type.is_error):
            error(self.pos, "Invalid operand types for '%s' (%s; %s)" %
7972
                (self.operator, self.operand1.type,
William Stein's avatar
William Stein committed
7973 7974 7975 7976
                    self.operand2.type))
        self.type = PyrexTypes.error_type


Robert Bradshaw's avatar
Robert Bradshaw committed
7977
class CBinopNode(BinopNode):
7978

Robert Bradshaw's avatar
Robert Bradshaw committed
7979 7980 7981 7982
    def analyse_types(self, env):
        BinopNode.analyse_types(self, env)
        if self.is_py_operation():
            self.type = PyrexTypes.error_type
7983

Stefan Behnel's avatar
Stefan Behnel committed
7984
    def py_operation_function(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
7985
        return ""
7986

Robert Bradshaw's avatar
Robert Bradshaw committed
7987 7988
    def calculate_result_code(self):
        return "(%s %s %s)" % (
7989 7990
            self.operand1.result(),
            self.operator,
Robert Bradshaw's avatar
Robert Bradshaw committed
7991 7992
            self.operand2.result())

7993 7994 7995 7996 7997 7998 7999 8000 8001 8002
    def compute_c_result_type(self, type1, type2):
        cpp_type = None
        if type1.is_cpp_class or type1.is_ptr:
            cpp_type = type1.find_cpp_operation_type(self.operator, type2)
        # FIXME: handle the reversed case?
        #if cpp_type is None and (type2.is_cpp_class or type2.is_ptr):
        #    cpp_type = type2.find_cpp_operation_type(self.operator, type1)
        # FIXME: do we need to handle other cases here?
        return cpp_type

Robert Bradshaw's avatar
Robert Bradshaw committed
8003 8004 8005 8006 8007 8008

def c_binop_constructor(operator):
    def make_binop_node(pos, **operands):
        return CBinopNode(pos, operator=operator, **operands)
    return make_binop_node

William Stein's avatar
William Stein committed
8009 8010
class NumBinopNode(BinopNode):
    #  Binary operation taking numeric arguments.
8011

Robert Bradshaw's avatar
Robert Bradshaw committed
8012
    infix = True
8013
    overflow_check = False
8014

William Stein's avatar
William Stein committed
8015 8016 8017 8018 8019 8020
    def analyse_c_operation(self, env):
        type1 = self.operand1.type
        type2 = self.operand2.type
        self.type = self.compute_c_result_type(type1, type2)
        if not self.type:
            self.type_error()
8021
            return
8022
        if self.type.is_complex:
Robert Bradshaw's avatar
Robert Bradshaw committed
8023
            self.infix = False
8024 8025 8026 8027 8028 8029 8030
        if (self.type.is_int
                and env.directives['overflowcheck']
                and self.operator in self.overflow_op_names):
            if (self.operator in ('+', '*')
                    and self.operand1.has_constant_result()
                    and not self.operand2.has_constant_result()):
                self.operand1, self.operand2 = self.operand2, self.operand1
8031
            self.overflow_check = True
8032
            self.func = self.type.overflow_check_binop(
8033
                self.overflow_op_names[self.operator],
8034 8035
                env,
                const_rhs = self.operand2.has_constant_result())
8036
            self.is_temp = True
8037
        if not self.infix or (type1.is_numeric and type2.is_numeric):
8038 8039
            self.operand1 = self.operand1.coerce_to(self.type, env)
            self.operand2 = self.operand2.coerce_to(self.type, env)
8040

William Stein's avatar
William Stein committed
8041 8042
    def compute_c_result_type(self, type1, type2):
        if self.c_types_okay(type1, type2):
8043 8044 8045 8046 8047
            widest_type = PyrexTypes.widest_numeric_type(type1, type2)
            if widest_type is PyrexTypes.c_bint_type:
                if self.operator not in '|^&':
                    # False + False == 0 # not False!
                    widest_type = PyrexTypes.c_int_type
8048 8049 8050
            else:
                widest_type = PyrexTypes.widest_numeric_type(
                    widest_type, PyrexTypes.c_int_type)
8051
            return widest_type
William Stein's avatar
William Stein committed
8052 8053
        else:
            return None
8054

8055 8056 8057 8058 8059 8060 8061 8062 8063 8064
    def may_be_none(self):
        type1 = self.operand1.type
        type2 = self.operand2.type
        if type1 and type1.is_builtin_type and type2 and type2.is_builtin_type:
            # XXX: I can't think of any case where a binary operation
            # on builtin types evaluates to None - add a special case
            # here if there is one.
            return False
        return super(NumBinopNode, self).may_be_none()

8065 8066 8067 8068 8069 8070 8071
    def get_constant_c_result_code(self):
        value1 = self.operand1.get_constant_c_result_code()
        value2 = self.operand2.get_constant_c_result_code()
        if value1 and value2:
            return "(%s %s %s)" % (value1, self.operator, value2)
        else:
            return None
8072

William Stein's avatar
William Stein committed
8073
    def c_types_okay(self, type1, type2):
8074 8075 8076
        #print "NumBinopNode.c_types_okay:", type1, type2 ###
        return (type1.is_numeric  or type1.is_enum) \
            and (type2.is_numeric  or type2.is_enum)
William Stein's avatar
William Stein committed
8077

8078 8079 8080 8081 8082 8083 8084 8085 8086 8087 8088 8089
    def generate_result_code(self, code):
        super(NumBinopNode, self).generate_result_code(code)
        if self.overflow_check:
            self.overflow_bit = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
            code.putln("%s = 0;" % self.overflow_bit);
            code.putln("%s = %s;" % (self.result(), self.calculate_result_code()))
            code.putln("if (unlikely(%s)) {" % self.overflow_bit)
            code.putln('PyErr_Format(PyExc_OverflowError, "value too large");')
            code.putln(code.error_goto(self.pos))
            code.putln("}")
            code.funcstate.release_temp(self.overflow_bit)

William Stein's avatar
William Stein committed
8090
    def calculate_result_code(self):
8091 8092 8093 8094 8095 8096 8097
        if self.overflow_check:
            return "%s(%s, %s, &%s)" % (
                self.func,
                self.operand1.result(),
                self.operand2.result(),
                self.overflow_bit)
        elif self.infix:
8098
            return "(%s %s %s)" % (
8099 8100
                self.operand1.result(),
                self.operator,
8101 8102
                self.operand2.result())
        else:
8103 8104 8105
            func = self.type.binary_op(self.operator)
            if func is None:
                error(self.pos, "binary operator %s not supported for %s" % (self.operator, self.type))
8106
            return "%s(%s, %s)" % (
8107
                func,
8108 8109
                self.operand1.result(),
                self.operand2.result())
8110

8111
    def is_py_operation_types(self, type1, type2):
Stefan Behnel's avatar
Stefan Behnel committed
8112 8113
        return (type1.is_unicode_char or
                type2.is_unicode_char or
8114
                BinopNode.is_py_operation_types(self, type1, type2))
8115

William Stein's avatar
William Stein committed
8116
    def py_operation_function(self):
8117 8118 8119 8120
        fuction = self.py_functions[self.operator]
        if self.inplace:
            fuction = fuction.replace('PyNumber_', 'PyNumber_InPlace')
        return fuction
William Stein's avatar
William Stein committed
8121 8122

    py_functions = {
Robert Bradshaw's avatar
Robert Bradshaw committed
8123 8124 8125
        "|":        "PyNumber_Or",
        "^":        "PyNumber_Xor",
        "&":        "PyNumber_And",
8126 8127
        "<<":       "PyNumber_Lshift",
        ">>":       "PyNumber_Rshift",
Robert Bradshaw's avatar
Robert Bradshaw committed
8128 8129 8130 8131
        "+":        "PyNumber_Add",
        "-":        "PyNumber_Subtract",
        "*":        "PyNumber_Multiply",
        "/":        "__Pyx_PyNumber_Divide",
8132
        "//":       "PyNumber_FloorDivide",
Robert Bradshaw's avatar
Robert Bradshaw committed
8133
        "%":        "PyNumber_Remainder",
8134
        "**":       "PyNumber_Power"
William Stein's avatar
William Stein committed
8135
    }
8136
    
8137
    overflow_op_names = {
8138 8139 8140
       "+":  "add",
       "-":  "sub",
       "*":  "mul",
8141
       "<<":  "lshift",
8142
    }
William Stein's avatar
William Stein committed
8143 8144 8145

class IntBinopNode(NumBinopNode):
    #  Binary operation taking integer arguments.
8146

William Stein's avatar
William Stein committed
8147
    def c_types_okay(self, type1, type2):
8148 8149 8150
        #print "IntBinopNode.c_types_okay:", type1, type2 ###
        return (type1.is_int or type1.is_enum) \
            and (type2.is_int or type2.is_enum)
William Stein's avatar
William Stein committed
8151

8152

William Stein's avatar
William Stein committed
8153 8154
class AddNode(NumBinopNode):
    #  '+' operator.
8155

8156 8157 8158
    def is_py_operation_types(self, type1, type2):
        if type1.is_string and type2.is_string:
            return 1
William Stein's avatar
William Stein committed
8159
        else:
8160
            return NumBinopNode.is_py_operation_types(self, type1, type2)
William Stein's avatar
William Stein committed
8161 8162

    def compute_c_result_type(self, type1, type2):
8163 8164
        #print "AddNode.compute_c_result_type:", type1, self.operator, type2 ###
        if (type1.is_ptr or type1.is_array) and (type2.is_int or type2.is_enum):
William Stein's avatar
William Stein committed
8165
            return type1
8166
        elif (type2.is_ptr or type2.is_array) and (type1.is_int or type1.is_enum):
William Stein's avatar
William Stein committed
8167 8168 8169 8170 8171 8172 8173 8174
            return type2
        else:
            return NumBinopNode.compute_c_result_type(
                self, type1, type2)


class SubNode(NumBinopNode):
    #  '-' operator.
8175

William Stein's avatar
William Stein committed
8176
    def compute_c_result_type(self, type1, type2):
8177
        if (type1.is_ptr or type1.is_array) and (type2.is_int or type2.is_enum):
William Stein's avatar
William Stein committed
8178
            return type1
8179
        elif (type1.is_ptr or type1.is_array) and (type2.is_ptr or type2.is_array):
William Stein's avatar
William Stein committed
8180 8181 8182 8183 8184 8185 8186 8187
            return PyrexTypes.c_int_type
        else:
            return NumBinopNode.compute_c_result_type(
                self, type1, type2)


class MulNode(NumBinopNode):
    #  '*' operator.
8188

8189
    def is_py_operation_types(self, type1, type2):
William Stein's avatar
William Stein committed
8190 8191 8192 8193
        if (type1.is_string and type2.is_int) \
            or (type2.is_string and type1.is_int):
                return 1
        else:
8194
            return NumBinopNode.is_py_operation_types(self, type1, type2)
William Stein's avatar
William Stein committed
8195 8196


8197 8198
class DivNode(NumBinopNode):
    #  '/' or '//' operator.
8199

8200
    cdivision = None
8201 8202
    truedivision = None   # == "unknown" if operator == '/'
    ctruedivision = False
Robert Bradshaw's avatar
Robert Bradshaw committed
8203
    cdivision_warnings = False
8204
    zerodivision_check = None
8205 8206 8207 8208 8209 8210 8211 8212 8213 8214 8215 8216 8217 8218 8219 8220 8221 8222 8223 8224 8225 8226

    def find_compile_time_binary_operator(self, op1, op2):
        func = compile_time_binary_operators[self.operator]
        if self.operator == '/' and self.truedivision is None:
            # => true div for floats, floor div for integers
            if isinstance(op1, (int,long)) and isinstance(op2, (int,long)):
                func = compile_time_binary_operators['//']
        return func

    def calculate_constant_result(self):
        op1 = self.operand1.constant_result
        op2 = self.operand2.constant_result
        func = self.find_compile_time_binary_operator(op1, op2)
        self.constant_result = func(
            self.operand1.constant_result,
            self.operand2.constant_result)

    def compile_time_value(self, denv):
        operand1 = self.operand1.compile_time_value(denv)
        operand2 = self.operand2.compile_time_value(denv)
        try:
            func = self.find_compile_time_binary_operator(
Robert Bradshaw's avatar
Robert Bradshaw committed
8227
                operand1, operand2)
8228 8229 8230 8231
            return func(operand1, operand2)
        except Exception, e:
            self.compile_time_value_error(e)

Robert Bradshaw's avatar
Robert Bradshaw committed
8232
    def analyse_operation(self, env):
8233 8234 8235 8236
        if self.cdivision or env.directives['cdivision']:
            self.ctruedivision = False
        else:
            self.ctruedivision = self.truedivision
Robert Bradshaw's avatar
Robert Bradshaw committed
8237
        NumBinopNode.analyse_operation(self, env)
8238 8239
        if self.is_cpp_operation():
            self.cdivision = True
8240
        if not self.type.is_pyobject:
8241 8242
            self.zerodivision_check = (
                self.cdivision is None and not env.directives['cdivision']
8243
                and (not self.operand2.has_constant_result() or
8244
                     self.operand2.constant_result == 0))
8245 8246 8247 8248
            if self.zerodivision_check or env.directives['cdivision_warnings']:
                # Need to check ahead of time to warn or raise zero division error
                self.operand1 = self.operand1.coerce_to_simple(env)
                self.operand2 = self.operand2.coerce_to_simple(env)
8249 8250
                if env.nogil:
                    error(self.pos, "Pythonic division not allowed without gil, consider using cython.cdivision(True)")
8251 8252 8253 8254 8255 8256 8257 8258 8259

    def compute_c_result_type(self, type1, type2):
        if self.operator == '/' and self.ctruedivision:
            if not type1.is_float and not type2.is_float:
                widest_type = PyrexTypes.widest_numeric_type(type1, PyrexTypes.c_double_type)
                widest_type = PyrexTypes.widest_numeric_type(type2, widest_type)
                return widest_type
        return NumBinopNode.compute_c_result_type(self, type1, type2)

8260 8261 8262 8263 8264
    def zero_division_message(self):
        if self.type.is_int:
            return "integer division or modulo by zero"
        else:
            return "float division"
Robert Bradshaw's avatar
Robert Bradshaw committed
8265

8266
    def generate_evaluation_code(self, code):
8267
        if not self.type.is_pyobject and not self.type.is_complex:
8268
            if self.cdivision is None:
8269
                self.cdivision = (code.globalstate.directives['cdivision']
8270 8271 8272
                                    or not self.type.signed
                                    or self.type.is_float)
            if not self.cdivision:
8273
                code.globalstate.use_utility_code(div_int_utility_code.specialize(self.type))
8274
        NumBinopNode.generate_evaluation_code(self, code)
8275
        self.generate_div_warning_code(code)
8276

8277
    def generate_div_warning_code(self, code):
8278 8279
        if not self.type.is_pyobject:
            if self.zerodivision_check:
8280 8281 8282 8283 8284
                if not self.infix:
                    zero_test = "%s(%s)" % (self.type.unary_op('zero'), self.operand2.result())
                else:
                    zero_test = "%s == 0" % self.operand2.result()
                code.putln("if (unlikely(%s)) {" % zero_test)
8285 8286 8287
                code.putln('PyErr_Format(PyExc_ZeroDivisionError, "%s");' % self.zero_division_message())
                code.putln(code.error_goto(self.pos))
                code.putln("}")
8288 8289 8290
                if self.type.is_int and self.type.signed and self.operator != '%':
                    code.globalstate.use_utility_code(division_overflow_test_code)
                    code.putln("else if (sizeof(%s) == sizeof(long) && unlikely(%s == -1) && unlikely(UNARY_NEG_WOULD_OVERFLOW(%s))) {" % (
8291
                                    self.type.declaration_code(''),
8292 8293 8294 8295 8296
                                    self.operand2.result(),
                                    self.operand1.result()))
                    code.putln('PyErr_Format(PyExc_OverflowError, "value too large to perform division");')
                    code.putln(code.error_goto(self.pos))
                    code.putln("}")
Robert Bradshaw's avatar
Robert Bradshaw committed
8297
            if code.globalstate.directives['cdivision_warnings'] and self.operator != '/':
8298 8299 8300 8301 8302
                code.globalstate.use_utility_code(cdivision_warning_utility_code)
                code.putln("if ((%s < 0) ^ (%s < 0)) {" % (
                                self.operand1.result(),
                                self.operand2.result()))
                code.putln(code.set_error_info(self.pos));
8303 8304 8305 8306 8307 8308
                code.put("if (__Pyx_cdivision_warning(%(FILENAME)s, "
                                                     "%(LINENO)s)) " % {
                    'FILENAME': Naming.filename_cname,
                    'LINENO':  Naming.lineno_cname,
                    })

8309 8310
                code.put_goto(code.error_label)
                code.putln("}")
8311

Robert Bradshaw's avatar
Robert Bradshaw committed
8312
    def calculate_result_code(self):
8313 8314 8315
        if self.type.is_complex:
            return NumBinopNode.calculate_result_code(self)
        elif self.type.is_float and self.operator == '//':
8316
            return "floor(%s / %s)" % (
8317
                self.operand1.result(),
8318
                self.operand2.result())
8319 8320 8321 8322 8323 8324 8325 8326 8327
        elif self.truedivision or self.cdivision:
            op1 = self.operand1.result()
            op2 = self.operand2.result()
            if self.truedivision:
                if self.type != self.operand1.type:
                    op1 = self.type.cast_code(op1)
                if self.type != self.operand2.type:
                    op2 = self.type.cast_code(op2)
            return "(%s / %s)" % (op1, op2)
8328 8329
        else:
            return "__Pyx_div_%s(%s, %s)" % (
Craig Citro's avatar
Craig Citro committed
8330
                    self.type.specialization_name(),
8331
                    self.operand1.result(),
8332
                    self.operand2.result())
Robert Bradshaw's avatar
Robert Bradshaw committed
8333 8334


Robert Bradshaw's avatar
Robert Bradshaw committed
8335
class ModNode(DivNode):
William Stein's avatar
William Stein committed
8336
    #  '%' operator.
8337

8338 8339 8340 8341
    def is_py_operation_types(self, type1, type2):
        return (type1.is_string
            or type2.is_string
            or NumBinopNode.is_py_operation_types(self, type1, type2))
William Stein's avatar
William Stein committed
8342

8343 8344 8345 8346 8347
    def zero_division_message(self):
        if self.type.is_int:
            return "integer division or modulo by zero"
        else:
            return "float divmod()"
8348

8349
    def generate_evaluation_code(self, code):
8350 8351 8352 8353 8354
        if not self.type.is_pyobject:
            if self.cdivision is None:
                self.cdivision = code.globalstate.directives['cdivision'] or not self.type.signed
            if not self.cdivision:
                if self.type.is_int:
8355
                    code.globalstate.use_utility_code(mod_int_utility_code.specialize(self.type))
8356
                else:
8357 8358
                    code.globalstate.use_utility_code(
                        mod_float_utility_code.specialize(self.type, math_h_modifier=self.type.math_h_modifier))
8359
        NumBinopNode.generate_evaluation_code(self, code)
8360
        self.generate_div_warning_code(code)
8361

Robert Bradshaw's avatar
Robert Bradshaw committed
8362
    def calculate_result_code(self):
8363 8364 8365 8366
        if self.cdivision:
            if self.type.is_float:
                return "fmod%s(%s, %s)" % (
                    self.type.math_h_modifier,
8367
                    self.operand1.result(),
8368 8369 8370
                    self.operand2.result())
            else:
                return "(%s %% %s)" % (
8371
                    self.operand1.result(),
8372
                    self.operand2.result())
Robert Bradshaw's avatar
Robert Bradshaw committed
8373
        else:
8374
            return "__Pyx_mod_%s(%s, %s)" % (
Craig Citro's avatar
Craig Citro committed
8375
                    self.type.specialization_name(),
8376
                    self.operand1.result(),
8377
                    self.operand2.result())
William Stein's avatar
William Stein committed
8378 8379 8380

class PowNode(NumBinopNode):
    #  '**' operator.
8381

Robert Bradshaw's avatar
Robert Bradshaw committed
8382 8383
    def analyse_c_operation(self, env):
        NumBinopNode.analyse_c_operation(self, env)
8384
        if self.type.is_complex:
Robert Bradshaw's avatar
Robert Bradshaw committed
8385 8386 8387 8388 8389 8390 8391
            if self.type.real_type.is_float:
                self.operand1 = self.operand1.coerce_to(self.type, env)
                self.operand2 = self.operand2.coerce_to(self.type, env)
                self.pow_func = "__Pyx_c_pow" + self.type.real_type.math_h_modifier
            else:
                error(self.pos, "complex int powers not supported")
                self.pow_func = "<error>"
8392
        elif self.type.is_float:
8393
            self.pow_func = "pow" + self.type.math_h_modifier
William Stein's avatar
William Stein committed
8394
        else:
Robert Bradshaw's avatar
Robert Bradshaw committed
8395 8396
            self.pow_func = "__Pyx_pow_%s" % self.type.declaration_code('').replace(' ', '_')
            env.use_utility_code(
8397
                    int_pow_utility_code.specialize(func_name=self.pow_func,
Robert Bradshaw's avatar
Robert Bradshaw committed
8398
                                                type=self.type.declaration_code('')))
8399

William Stein's avatar
William Stein committed
8400
    def calculate_result_code(self):
8401 8402 8403 8404 8405 8406
        # Work around MSVC overloading ambiguity.
        def typecast(operand):
            if self.type == operand.type:
                return operand.result()
            else:
                return self.type.cast_code(operand.result())
Robert Bradshaw's avatar
Robert Bradshaw committed
8407
        return "%s(%s, %s)" % (
8408 8409
            self.pow_func,
            typecast(self.operand1),
8410
            typecast(self.operand2))
8411

William Stein's avatar
William Stein committed
8412

Craig Citro's avatar
Craig Citro committed
8413
# Note: This class is temporarily "shut down" into an ineffective temp
8414 8415
# allocation mode.
#
Craig Citro's avatar
Craig Citro committed
8416 8417 8418
# More sophisticated temp reuse was going on before, one could have a
# look at adding this again after /all/ classes are converted to the
# new temp scheme. (The temp juggling cannot work otherwise).
8419
class BoolBinopNode(ExprNode):
William Stein's avatar
William Stein committed
8420 8421 8422 8423 8424
    #  Short-circuiting boolean operation.
    #
    #  operator     string
    #  operand1     ExprNode
    #  operand2     ExprNode
8425

8426
    subexprs = ['operand1', 'operand2']
8427

8428
    def infer_type(self, env):
8429 8430
        type1 = self.operand1.infer_type(env)
        type2 = self.operand2.infer_type(env)
8431
        return PyrexTypes.independent_spanning_type(type1, type2)
8432

Stefan Behnel's avatar
Stefan Behnel committed
8433 8434 8435 8436 8437 8438
    def may_be_none(self):
        if self.operator == 'or':
            return self.operand2.may_be_none()
        else:
            return self.operand1.may_be_none() or self.operand2.may_be_none()

8439 8440 8441 8442 8443 8444 8445 8446 8447
    def calculate_constant_result(self):
        if self.operator == 'and':
            self.constant_result = \
                self.operand1.constant_result and \
                self.operand2.constant_result
        else:
            self.constant_result = \
                self.operand1.constant_result or \
                self.operand2.constant_result
8448

8449 8450 8451 8452 8453 8454 8455
    def compile_time_value(self, denv):
        if self.operator == 'and':
            return self.operand1.compile_time_value(denv) \
                and self.operand2.compile_time_value(denv)
        else:
            return self.operand1.compile_time_value(denv) \
                or self.operand2.compile_time_value(denv)
8456

8457
    def coerce_to_boolean(self, env):
8458 8459 8460 8461 8462 8463 8464
        return BoolBinopNode(
            self.pos,
            operator = self.operator,
            operand1 = self.operand1.coerce_to_boolean(env),
            operand2 = self.operand2.coerce_to_boolean(env),
            type = PyrexTypes.c_bint_type,
            is_temp = self.is_temp)
8465

William Stein's avatar
William Stein committed
8466 8467 8468
    def analyse_types(self, env):
        self.operand1.analyse_types(env)
        self.operand2.analyse_types(env)
8469
        self.type = PyrexTypes.independent_spanning_type(self.operand1.type, self.operand2.type)
8470 8471
        self.operand1 = self.operand1.coerce_to(self.type, env)
        self.operand2 = self.operand2.coerce_to(self.type, env)
8472

William Stein's avatar
William Stein committed
8473 8474
        # For what we're about to do, it's vital that
        # both operands be temp nodes.
8475 8476
        self.operand1 = self.operand1.coerce_to_simple(env)
        self.operand2 = self.operand2.coerce_to_simple(env)
William Stein's avatar
William Stein committed
8477
        self.is_temp = 1
8478 8479 8480

    gil_message = "Truth-testing Python object"

William Stein's avatar
William Stein committed
8481
    def check_const(self):
8482
        return self.operand1.check_const() and self.operand2.check_const()
8483

William Stein's avatar
William Stein committed
8484
    def generate_evaluation_code(self, code):
8485
        code.mark_pos(self.pos)
William Stein's avatar
William Stein committed
8486
        self.operand1.generate_evaluation_code(code)
8487
        test_result, uses_temp = self.generate_operand1_test(code)
William Stein's avatar
William Stein committed
8488 8489 8490 8491 8492 8493 8494 8495
        if self.operator == 'and':
            sense = ""
        else:
            sense = "!"
        code.putln(
            "if (%s%s) {" % (
                sense,
                test_result))
8496 8497
        if uses_temp:
            code.funcstate.release_temp(test_result)
8498
        self.operand1.generate_disposal_code(code)
William Stein's avatar
William Stein committed
8499
        self.operand2.generate_evaluation_code(code)
8500
        self.allocate_temp_result(code)
8501
        self.operand2.make_owned_reference(code)
8502
        code.putln("%s = %s;" % (self.result(), self.operand2.result()))
8503 8504
        self.operand2.generate_post_assignment_code(code)
        self.operand2.free_temps(code)
8505
        code.putln("} else {")
8506
        self.operand1.make_owned_reference(code)
8507
        code.putln("%s = %s;" % (self.result(), self.operand1.result()))
8508 8509
        self.operand1.generate_post_assignment_code(code)
        self.operand1.free_temps(code)
8510
        code.putln("}")
8511

William Stein's avatar
William Stein committed
8512 8513 8514
    def generate_operand1_test(self, code):
        #  Generate code to test the truth of the first operand.
        if self.type.is_pyobject:
8515 8516
            test_result = code.funcstate.allocate_temp(PyrexTypes.c_bint_type,
                                                       manage_ref=False)
William Stein's avatar
William Stein committed
8517
            code.putln(
8518
                "%s = __Pyx_PyObject_IsTrue(%s); %s" % (
William Stein's avatar
William Stein committed
8519 8520
                    test_result,
                    self.operand1.py_result(),
Robert Bradshaw's avatar
Robert Bradshaw committed
8521
                    code.error_goto_if_neg(test_result, self.pos)))
William Stein's avatar
William Stein committed
8522
        else:
8523
            test_result = self.operand1.result()
8524
        return (test_result, self.type.is_pyobject)
William Stein's avatar
William Stein committed
8525 8526


8527
class CondExprNode(ExprNode):
Robert Bradshaw's avatar
Robert Bradshaw committed
8528 8529 8530 8531 8532
    #  Short-circuiting conditional expression.
    #
    #  test        ExprNode
    #  true_val    ExprNode
    #  false_val   ExprNode
8533

8534 8535
    true_val = None
    false_val = None
8536

Robert Bradshaw's avatar
Robert Bradshaw committed
8537
    subexprs = ['test', 'true_val', 'false_val']
8538

Robert Bradshaw's avatar
Robert Bradshaw committed
8539 8540
    def type_dependencies(self, env):
        return self.true_val.type_dependencies(env) + self.false_val.type_dependencies(env)
8541

Robert Bradshaw's avatar
Robert Bradshaw committed
8542
    def infer_type(self, env):
8543 8544
        return PyrexTypes.independent_spanning_type(self.true_val.infer_type(env),
                                                    self.false_val.infer_type(env))
8545 8546 8547 8548 8549 8550 8551

    def calculate_constant_result(self):
        if self.test.constant_result:
            self.constant_result = self.true_val.constant_result
        else:
            self.constant_result = self.false_val.constant_result

Robert Bradshaw's avatar
Robert Bradshaw committed
8552 8553 8554 8555 8556
    def analyse_types(self, env):
        self.test.analyse_types(env)
        self.test = self.test.coerce_to_boolean(env)
        self.true_val.analyse_types(env)
        self.false_val.analyse_types(env)
8557
        self.type = PyrexTypes.independent_spanning_type(self.true_val.type, self.false_val.type)
8558 8559 8560 8561 8562
        if self.true_val.type.is_pyobject or self.false_val.type.is_pyobject:
            self.true_val = self.true_val.coerce_to(self.type, env)
            self.false_val = self.false_val.coerce_to(self.type, env)
        self.is_temp = 1
        if self.type == PyrexTypes.error_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
8563
            self.type_error()
8564

Robert Bradshaw's avatar
Robert Bradshaw committed
8565 8566 8567 8568 8569
    def type_error(self):
        if not (self.true_val.type.is_error or self.false_val.type.is_error):
            error(self.pos, "Incompatable types in conditional expression (%s; %s)" %
                (self.true_val.type, self.false_val.type))
        self.type = PyrexTypes.error_type
8570

Robert Bradshaw's avatar
Robert Bradshaw committed
8571
    def check_const(self):
8572
        return (self.test.check_const()
8573 8574
            and self.true_val.check_const()
            and self.false_val.check_const())
8575

Robert Bradshaw's avatar
Robert Bradshaw committed
8576
    def generate_evaluation_code(self, code):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8577 8578
        # Because subexprs may not be evaluated we can use a more optimal
        # subexpr allocation strategy than the default, so override evaluation_code.
8579

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8580
        code.mark_pos(self.pos)
8581
        self.allocate_temp_result(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
8582
        self.test.generate_evaluation_code(code)
8583
        code.putln("if (%s) {" % self.test.result() )
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8584
        self.eval_and_get(code, self.true_val)
Robert Bradshaw's avatar
Robert Bradshaw committed
8585
        code.putln("} else {")
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8586
        self.eval_and_get(code, self.false_val)
Robert Bradshaw's avatar
Robert Bradshaw committed
8587 8588
        code.putln("}")
        self.test.generate_disposal_code(code)
8589
        self.test.free_temps(code)
Robert Bradshaw's avatar
Robert Bradshaw committed
8590

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8591 8592 8593
    def eval_and_get(self, code, expr):
        expr.generate_evaluation_code(code)
        expr.make_owned_reference(code)
8594
        code.putln('%s = %s;' % (self.result(), expr.result_as(self.ctype())))
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
8595 8596 8597
        expr.generate_post_assignment_code(code)
        expr.free_temps(code)

8598 8599 8600 8601 8602 8603 8604 8605
richcmp_constants = {
    "<" : "Py_LT",
    "<=": "Py_LE",
    "==": "Py_EQ",
    "!=": "Py_NE",
    "<>": "Py_NE",
    ">" : "Py_GT",
    ">=": "Py_GE",
8606 8607 8608
    # the following are faked by special compare functions
    "in"    : "Py_EQ",
    "not_in": "Py_NE",
8609 8610
}

8611
class CmpNode(object):
William Stein's avatar
William Stein committed
8612 8613
    #  Mixin class containing code common to PrimaryCmpNodes
    #  and CascadedCmpNodes.
8614 8615

    special_bool_cmp_function = None
8616
    special_bool_cmp_utility_code = None
8617

Stefan Behnel's avatar
typo  
Stefan Behnel committed
8618
    def infer_type(self, env):
8619 8620
        # TODO: Actually implement this (after merging with -unstable).
        return py_object_type
8621 8622 8623 8624 8625

    def calculate_cascaded_constant_result(self, operand1_result):
        func = compile_time_binary_operators[self.operator]
        operand2_result = self.operand2.constant_result
        result = func(operand1_result, operand2_result)
8626 8627 8628 8629 8630 8631 8632
        if self.cascade:
            self.cascade.calculate_cascaded_constant_result(operand2_result)
            if self.cascade.constant_result:
                self.constant_result = result and self.cascade.constant_result
        else:
            self.constant_result = result

8633 8634
    def cascaded_compile_time_value(self, operand1, denv):
        func = get_compile_time_binop(self)
8635
        operand2 = self.operand2.compile_time_value(denv)
8636 8637 8638 8639
        try:
            result = func(operand1, operand2)
        except Exception, e:
            self.compile_time_value_error(e)
8640
            result = None
8641 8642 8643
        if result:
            cascade = self.cascade
            if cascade:
8644
                # FIXME: I bet this must call cascaded_compile_time_value()
8645
                result = result and cascade.cascaded_compile_time_value(operand2, denv)
8646 8647
        return result

8648
    def is_cpp_comparison(self):
Robert Bradshaw's avatar
Robert Bradshaw committed
8649
        return self.operand1.type.is_cpp_class or self.operand2.type.is_cpp_class
8650

8651
    def find_common_int_type(self, env, op, operand1, operand2):
8652 8653 8654 8655 8656 8657
        # type1 != type2 and at least one of the types is not a C int
        type1 = operand1.type
        type2 = operand2.type
        type1_can_be_int = False
        type2_can_be_int = False

8658
        if operand1.is_string_literal and operand1.can_coerce_to_char_literal():
8659
            type1_can_be_int = True
8660
        if operand2.is_string_literal and operand2.can_coerce_to_char_literal():
8661 8662 8663 8664
            type2_can_be_int = True

        if type1.is_int:
            if type2_can_be_int:
8665
                return type1
8666 8667
        elif type2.is_int:
            if type1_can_be_int:
8668
                return type2
8669 8670
        elif type1_can_be_int:
            if type2_can_be_int:
8671
                return PyrexTypes.c_uchar_type
William Stein's avatar
William Stein committed
8672

8673
        return None
8674

8675
    def find_common_type(self, env, op, operand1, common_type=None):
8676
        operand2 = self.operand2
William Stein's avatar
William Stein committed
8677 8678
        type1 = operand1.type
        type2 = operand2.type
8679

8680 8681
        new_common_type = None

Stefan Behnel's avatar
Stefan Behnel committed
8682
        # catch general errors
8683 8684 8685
        if type1 == str_type and (type2.is_string or type2 in (bytes_type, unicode_type)) or \
               type2 == str_type and (type1.is_string or type1 in (bytes_type, unicode_type)):
            error(self.pos, "Comparisons between bytes/unicode and str are not portable to Python 3")
8686
            new_common_type = error_type
Stefan Behnel's avatar
Stefan Behnel committed
8687 8688

        # try to use numeric comparisons where possible
8689
        elif type1.is_complex or type2.is_complex:
8690 8691 8692
            if op not in ('==', '!=') \
               and (type1.is_complex or type1.is_numeric) \
               and (type2.is_complex or type2.is_numeric):
8693 8694
                error(self.pos, "complex types are unordered")
                new_common_type = error_type
8695
            elif type1.is_pyobject:
8696 8697 8698
                new_common_type = type1
            elif type2.is_pyobject:
                new_common_type = type2
8699
            else:
8700
                new_common_type = PyrexTypes.widest_numeric_type(type1, type2)
8701 8702
        elif type1.is_numeric and type2.is_numeric:
            new_common_type = PyrexTypes.widest_numeric_type(type1, type2)
8703
        elif common_type is None or not common_type.is_pyobject:
8704
            new_common_type = self.find_common_int_type(env, op, operand1, operand2)
8705 8706

        if new_common_type is None:
Stefan Behnel's avatar
Stefan Behnel committed
8707
            # fall back to generic type compatibility tests
8708
            if type1 == type2:
8709 8710 8711 8712 8713 8714
                new_common_type = type1
            elif type1.is_pyobject or type2.is_pyobject:
                if type2.is_numeric or type2.is_string:
                    if operand2.check_for_coercion_error(type1):
                        new_common_type = error_type
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
8715
                        new_common_type = py_object_type
8716 8717 8718 8719
                elif type1.is_numeric or type1.is_string:
                    if operand1.check_for_coercion_error(type2):
                        new_common_type = error_type
                    else:
Robert Bradshaw's avatar
Robert Bradshaw committed
8720 8721 8722
                        new_common_type = py_object_type
                elif py_object_type.assignable_from(type1) and py_object_type.assignable_from(type2):
                    new_common_type = py_object_type
8723 8724 8725 8726
                else:
                    # one Python type and one non-Python type, not assignable
                    self.invalid_types_error(operand1, op, operand2)
                    new_common_type = error_type
8727 8728 8729 8730
            elif type1.assignable_from(type2):
                new_common_type = type1
            elif type2.assignable_from(type1):
                new_common_type = type2
8731 8732 8733 8734
            else:
                # C types that we couldn't handle up to here are an error
                self.invalid_types_error(operand1, op, operand2)
                new_common_type = error_type
8735

8736 8737 8738 8739 8740 8741
        if new_common_type.is_string and (isinstance(operand1, BytesNode) or
                                          isinstance(operand2, BytesNode)):
            # special case when comparing char* to bytes literal: must
            # compare string values!
            new_common_type = bytes_type

Stefan Behnel's avatar
Stefan Behnel committed
8742
        # recursively merge types
8743
        if common_type is None or new_common_type.is_error:
8744
            common_type = new_common_type
William Stein's avatar
William Stein committed
8745
        else:
8746 8747 8748
            # we could do a lot better by splitting the comparison
            # into a non-Python part and a Python part, but this is
            # safer for now
8749
            common_type = PyrexTypes.spanning_type(common_type, new_common_type)
8750 8751

        if self.cascade:
8752
            common_type = self.cascade.find_common_type(env, self.operator, operand2, common_type)
8753

8754 8755
        return common_type

8756 8757 8758 8759
    def invalid_types_error(self, operand1, op, operand2):
        error(self.pos, "Invalid types for '%s' (%s, %s)" %
              (op, operand1.type, operand2.type))

Stefan Behnel's avatar
Stefan Behnel committed
8760
    def is_python_comparison(self):
8761 8762 8763 8764 8765
        return (not self.is_ptr_contains()
            and not self.is_c_string_contains()
            and (self.has_python_operands()
                 or (self.cascade and self.cascade.is_python_comparison())
                 or self.operator in ('in', 'not_in')))
Stefan Behnel's avatar
Stefan Behnel committed
8766

8767 8768 8769 8770 8771 8772
    def coerce_operands_to(self, dst_type, env):
        operand2 = self.operand2
        if operand2.type != dst_type:
            self.operand2 = operand2.coerce_to(dst_type, env)
        if self.cascade:
            self.cascade.coerce_operands_to(dst_type, env)
8773

8774
    def is_python_result(self):
8775
        return ((self.has_python_operands() and
8776
                 self.special_bool_cmp_function is None and
8777
                 self.operator not in ('is', 'is_not', 'in', 'not_in') and
8778 8779
                 not self.is_c_string_contains() and
                 not self.is_ptr_contains())
8780
            or (self.cascade and self.cascade.is_python_result()))
William Stein's avatar
William Stein committed
8781

8782 8783
    def is_c_string_contains(self):
        return self.operator in ('in', 'not_in') and \
8784 8785
               ((self.operand1.type.is_int
                 and (self.operand2.type.is_string or self.operand2.type is bytes_type)) or
Stefan Behnel's avatar
Stefan Behnel committed
8786
                (self.operand1.type.is_unicode_char
8787
                 and self.operand2.type is unicode_type))
8788

8789 8790
    def is_ptr_contains(self):
        if self.operator in ('in', 'not_in'):
8791 8792 8793
            container_type = self.operand2.type
            return (container_type.is_ptr or container_type.is_array) \
                and not container_type.is_string
8794

8795
    def find_special_bool_compare_function(self, env, operand1):
8796
        # note: currently operand1 must get coerced to a Python object if we succeed here!
8797
        if self.operator in ('==', '!='):
8798
            type1, type2 = operand1.type, self.operand2.type
8799
            if type1.is_builtin_type and type2.is_builtin_type:
8800
                if type1 is Builtin.unicode_type or type2 is Builtin.unicode_type:
8801
                    self.special_bool_cmp_utility_code = UtilityCode.load_cached("UnicodeEquals", "StringTools.c")
8802 8803
                    self.special_bool_cmp_function = "__Pyx_PyUnicode_Equals"
                    return True
8804
                elif type1 is Builtin.bytes_type or type2 is Builtin.bytes_type:
8805
                    self.special_bool_cmp_utility_code = UtilityCode.load_cached("BytesEquals", "StringTools.c")
8806 8807 8808
                    self.special_bool_cmp_function = "__Pyx_PyBytes_Equals"
                    return True
                elif type1 is Builtin.str_type or type2 is Builtin.str_type:
8809
                    self.special_bool_cmp_utility_code = UtilityCode.load_cached("StrEquals", "StringTools.c")
8810 8811
                    self.special_bool_cmp_function = "__Pyx_PyString_Equals"
                    return True
8812 8813 8814 8815 8816 8817
        elif self.operator in ('in', 'not_in'):
            if self.operand2.type is Builtin.dict_type:
                self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable")
                self.special_bool_cmp_utility_code = UtilityCode.load_cached("PyDictContains", "ObjectHandling.c")
                self.special_bool_cmp_function = "__Pyx_PyDict_Contains"
                return True
8818 8819 8820
            else:
                if not self.operand2.type.is_pyobject:
                    self.operand2 = self.operand2.coerce_to_pyobject(env)
8821 8822 8823
                self.special_bool_cmp_utility_code = UtilityCode.load_cached("PySequenceContains", "ObjectHandling.c")
                self.special_bool_cmp_function = "__Pyx_PySequence_Contains"
                return True
8824 8825
        return False

8826
    def generate_operation_code(self, code, result_code,
William Stein's avatar
William Stein committed
8827
            operand1, op , operand2):
8828
        if self.type.is_pyobject:
8829 8830 8831 8832 8833 8834 8835 8836
            error_clause = code.error_goto_if_null
            got_ref = "__Pyx_XGOTREF(%s); " % result_code
            if self.special_bool_cmp_function:
                code.globalstate.use_utility_code(
                    UtilityCode.load_cached("PyBoolOrNullFromLong", "ObjectHandling.c"))
                coerce_result = "__Pyx_PyBoolOrNull_FromLong"
            else:
                coerce_result = "__Pyx_PyBool_FromLong"
8837
        else:
8838 8839
            error_clause = code.error_goto_if_neg
            got_ref = ""
8840
            coerce_result = ""
8841

8842 8843 8844 8845 8846 8847 8848 8849 8850
        if self.special_bool_cmp_function:
            if operand1.type.is_pyobject:
                result1 = operand1.py_result()
            else:
                result1 = operand1.result()
            if operand2.type.is_pyobject:
                result2 = operand2.py_result()
            else:
                result2 = operand2.result()
8851 8852
            if self.special_bool_cmp_utility_code:
                code.globalstate.use_utility_code(self.special_bool_cmp_utility_code)
8853
            code.putln(
8854
                "%s = %s(%s(%s, %s, %s)); %s%s" % (
8855 8856
                    result_code,
                    coerce_result,
8857 8858 8859 8860 8861 8862 8863 8864 8865
                    self.special_bool_cmp_function,
                    result1, result2, richcmp_constants[op],
                    got_ref,
                    error_clause(result_code, self.pos)))

        elif operand1.type.is_pyobject and op not in ('is', 'is_not'):
            assert op not in ('in', 'not_in'), op
            code.putln("%s = PyObject_RichCompare(%s, %s, %s); %s%s" % (
                    result_code,
8866
                    operand1.py_result(),
8867 8868
                    operand2.py_result(),
                    richcmp_constants[op],
8869 8870
                    got_ref,
                    error_clause(result_code, self.pos)))
8871

8872
        elif operand1.type.is_complex:
8873
            code.putln("%s = %s(%s%s(%s, %s));" % (
8874
                result_code,
8875
                coerce_result,
8876
                op == "!=" and "!" or "",
8877 8878
                operand1.type.unary_op('eq'),
                operand1.result(),
8879
                operand2.result()))
8880

William Stein's avatar
William Stein committed
8881
        else:
8882 8883 8884 8885 8886
            type1 = operand1.type
            type2 = operand2.type
            if (type1.is_extension_type or type2.is_extension_type) \
                    and not type1.same_as(type2):
                common_type = py_object_type
8887 8888
            elif type1.is_numeric:
                common_type = PyrexTypes.widest_numeric_type(type1, type2)
8889
            else:
8890 8891 8892
                common_type = type1
            code1 = operand1.result_as(common_type)
            code2 = operand2.result_as(common_type)
8893
            code.putln("%s = %s(%s %s %s);" % (
8894 8895 8896 8897
                result_code,
                coerce_result,
                code1,
                self.c_operator(op),
8898 8899
                code2))

William Stein's avatar
William Stein committed
8900 8901 8902 8903 8904 8905 8906
    def c_operator(self, op):
        if op == 'is':
            return "=="
        elif op == 'is_not':
            return "!="
        else:
            return op
8907

8908
class PrimaryCmpNode(ExprNode, CmpNode):
William Stein's avatar
William Stein committed
8909 8910 8911 8912 8913 8914 8915
    #  Non-cascaded comparison or first comparison of
    #  a cascaded sequence.
    #
    #  operator      string
    #  operand1      ExprNode
    #  operand2      ExprNode
    #  cascade       CascadedCmpNode
8916

William Stein's avatar
William Stein committed
8917 8918 8919 8920
    #  We don't use the subexprs mechanism, because
    #  things here are too complicated for it to handle.
    #  Instead, we override all the framework methods
    #  which use it.
8921

8922
    child_attrs = ['operand1', 'operand2', 'coerced_operand2', 'cascade']
8923

William Stein's avatar
William Stein committed
8924
    cascade = None
8925
    coerced_operand2 = None
8926
    is_memslice_nonecheck = False
8927

Robert Bradshaw's avatar
Robert Bradshaw committed
8928 8929 8930 8931 8932 8933 8934
    def infer_type(self, env):
        # TODO: Actually implement this (after merging with -unstable).
        return py_object_type

    def type_dependencies(self, env):
        return ()

8935
    def calculate_constant_result(self):
8936
        self.calculate_cascaded_constant_result(self.operand1.constant_result)
8937

8938
    def compile_time_value(self, denv):
8939
        operand1 = self.operand1.compile_time_value(denv)
8940 8941
        return self.cascaded_compile_time_value(operand1, denv)

William Stein's avatar
William Stein committed
8942 8943 8944
    def analyse_types(self, env):
        self.operand1.analyse_types(env)
        self.operand2.analyse_types(env)
8945 8946
        if self.is_cpp_comparison():
            self.analyse_cpp_comparison(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
8947 8948 8949
            if self.cascade:
                error(self.pos, "Cascading comparison not yet supported for cpp types.")
            return
8950 8951 8952 8953

        if self.analyse_memoryviewslice_comparison(env):
            return

William Stein's avatar
William Stein committed
8954
        if self.cascade:
8955 8956
            self.cascade.analyse_types(env)

8957
        if self.operator in ('in', 'not_in'):
8958 8959 8960 8961 8962 8963 8964
            if self.is_c_string_contains():
                self.is_pycmp = False
                common_type = None
                if self.cascade:
                    error(self.pos, "Cascading comparison not yet supported for 'int_val in string'.")
                    return
                if self.operand2.type is unicode_type:
8965
                    env.use_utility_code(UtilityCode.load_cached("PyUCS4InUnicode", "StringTools.c"))
8966 8967 8968 8969 8970
                else:
                    if self.operand1.type is PyrexTypes.c_uchar_type:
                        self.operand1 = self.operand1.coerce_to(PyrexTypes.c_char_type, env)
                    if self.operand2.type is not bytes_type:
                        self.operand2 = self.operand2.coerce_to(bytes_type, env)
8971
                    env.use_utility_code(UtilityCode.load_cached("BytesContains", "StringTools.c"))
Stefan Behnel's avatar
Stefan Behnel committed
8972 8973
                self.operand2 = self.operand2.as_none_safe_node(
                    "argument of type 'NoneType' is not iterable")
8974 8975
            elif self.is_ptr_contains():
                if self.cascade:
Stefan Behnel's avatar
Stefan Behnel committed
8976
                    error(self.pos, "Cascading comparison not supported for 'val in sliced pointer'.")
8977 8978 8979
                self.type = PyrexTypes.c_bint_type
                # Will be transformed by IterationTransform
                return
8980 8981 8982 8983 8984
            elif self.find_special_bool_compare_function(env, self.operand1):
                if not self.operand1.type.is_pyobject:
                    self.operand1 = self.operand1.coerce_to_pyobject(env)
                common_type = None # if coercion needed, the method call above has already done it
                self.is_pycmp = False # result is bint
8985 8986 8987
            else:
                common_type = py_object_type
                self.is_pycmp = True
8988
        elif self.find_special_bool_compare_function(env, self.operand1):
8989 8990
            if not self.operand1.type.is_pyobject:
                self.operand1 = self.operand1.coerce_to_pyobject(env)
8991 8992
            common_type = None # if coercion needed, the method call above has already done it
            self.is_pycmp = False # result is bint
8993 8994 8995 8996
        else:
            common_type = self.find_common_type(env, self.operator, self.operand1)
            self.is_pycmp = common_type.is_pyobject

8997
        if common_type is not None and not common_type.is_error:
8998 8999 9000
            if self.operand1.type != common_type:
                self.operand1 = self.operand1.coerce_to(common_type, env)
            self.coerce_operands_to(common_type, env)
9001

William Stein's avatar
William Stein committed
9002
        if self.cascade:
9003
            self.operand2 = self.operand2.coerce_to_simple(env)
William Stein's avatar
William Stein committed
9004
            self.cascade.coerce_cascaded_operands_to_temp(env)
9005 9006 9007
            operand2 = self.cascade.optimise_comparison(self.operand2, env)
            if operand2 is not self.operand2:
                self.coerced_operand2 = operand2
9008 9009 9010 9011 9012 9013 9014 9015
        if self.is_python_result():
            self.type = PyrexTypes.py_object_type
        else:
            self.type = PyrexTypes.c_bint_type
        cdr = self.cascade
        while cdr:
            cdr.type = self.type
            cdr = cdr.cascade
9016 9017
        if self.is_pycmp or self.cascade or self.special_bool_cmp_function:
            # 1) owned reference, 2) reused value, 3) potential function error return value
William Stein's avatar
William Stein committed
9018
            self.is_temp = 1
9019

9020 9021 9022
    def analyse_cpp_comparison(self, env):
        type1 = self.operand1.type
        type2 = self.operand2.type
9023 9024
        entry = env.lookup_operator(self.operator, [self.operand1, self.operand2])
        if entry is None:
9025 9026
            error(self.pos, "Invalid types for '%s' (%s, %s)" %
                (self.operator, type1, type2))
9027 9028 9029
            self.type = PyrexTypes.error_type
            self.result_code = "<error>"
            return
9030 9031 9032 9033 9034
        func_type = entry.type
        if func_type.is_ptr:
            func_type = func_type.base_type
        if len(func_type.args) == 1:
            self.operand2 = self.operand2.coerce_to(func_type.args[0].type, env)
9035
        else:
9036 9037 9038
            self.operand1 = self.operand1.coerce_to(func_type.args[0].type, env)
            self.operand2 = self.operand2.coerce_to(func_type.args[1].type, env)
        self.type = func_type.return_type
9039

9040 9041 9042 9043 9044 9045 9046 9047 9048 9049 9050 9051 9052
    def analyse_memoryviewslice_comparison(self, env):
        have_none = self.operand1.is_none or self.operand2.is_none
        have_slice = (self.operand1.type.is_memoryviewslice or
                      self.operand2.type.is_memoryviewslice)
        ops = ('==', '!=', 'is', 'is_not')
        if have_slice and have_none and self.operator in ops:
            self.is_pycmp = False
            self.type = PyrexTypes.c_bint_type
            self.is_memslice_nonecheck = True
            return True

        return False

William Stein's avatar
William Stein committed
9053 9054 9055
    def has_python_operands(self):
        return (self.operand1.type.is_pyobject
            or self.operand2.type.is_pyobject)
9056

William Stein's avatar
William Stein committed
9057 9058 9059
    def check_const(self):
        if self.cascade:
            self.not_const()
9060 9061 9062
            return False
        else:
            return self.operand1.check_const() and self.operand2.check_const()
William Stein's avatar
William Stein committed
9063 9064

    def calculate_result_code(self):
9065 9066 9067 9068 9069 9070 9071
        if self.operand1.type.is_complex:
            if self.operator == "!=":
                negation = "!"
            else:
                negation = ""
            return "(%s%s(%s, %s))" % (
                negation,
9072 9073
                self.operand1.type.binary_op('=='),
                self.operand1.result(),
9074
                self.operand2.result())
9075
        elif self.is_c_string_contains():
9076
            if self.operand2.type is unicode_type:
9077
                method = "__Pyx_UnicodeContainsUCS4"
9078
            else:
9079
                method = "__Pyx_BytesContains"
9080 9081 9082 9083 9084 9085 9086
            if self.operator == "not_in":
                negation = "!"
            else:
                negation = ""
            return "(%s%s(%s, %s))" % (
                negation,
                method,
9087
                self.operand2.result(),
9088
                self.operand1.result())
9089
        else:
9090 9091 9092 9093 9094 9095 9096 9097
            result1 = self.operand1.result()
            result2 = self.operand2.result()
            if self.is_memslice_nonecheck:
                if self.operand1.type.is_memoryviewslice:
                    result1 = "((PyObject *) %s.memview)" % result1
                else:
                    result2 = "((PyObject *) %s.memview)" % result2

9098
            return "(%s %s %s)" % (
9099
                result1,
9100
                self.c_operator(self.operator),
9101
                result2)
9102

William Stein's avatar
William Stein committed
9103 9104 9105 9106
    def generate_evaluation_code(self, code):
        self.operand1.generate_evaluation_code(code)
        self.operand2.generate_evaluation_code(code)
        if self.is_temp:
9107
            self.allocate_temp_result(code)
9108
            self.generate_operation_code(code, self.result(),
William Stein's avatar
William Stein committed
9109 9110
                self.operand1, self.operator, self.operand2)
            if self.cascade:
9111 9112 9113
                self.cascade.generate_evaluation_code(
                    code, self.result(), self.coerced_operand2 or self.operand2,
                    needs_evaluation=self.coerced_operand2 is not None)
William Stein's avatar
William Stein committed
9114
            self.operand1.generate_disposal_code(code)
9115
            self.operand1.free_temps(code)
William Stein's avatar
William Stein committed
9116
            self.operand2.generate_disposal_code(code)
9117
            self.operand2.free_temps(code)
9118

William Stein's avatar
William Stein committed
9119 9120 9121 9122 9123
    def generate_subexpr_disposal_code(self, code):
        #  If this is called, it is a non-cascaded cmp,
        #  so only need to dispose of the two main operands.
        self.operand1.generate_disposal_code(code)
        self.operand2.generate_disposal_code(code)
9124

9125 9126 9127 9128 9129
    def free_subexpr_temps(self, code):
        #  If this is called, it is a non-cascaded cmp,
        #  so only need to dispose of the two main operands.
        self.operand1.free_temps(code)
        self.operand2.free_temps(code)
9130

9131 9132 9133 9134 9135
    def annotate(self, code):
        self.operand1.annotate(code)
        self.operand2.annotate(code)
        if self.cascade:
            self.cascade.annotate(code)
William Stein's avatar
William Stein committed
9136 9137 9138


class CascadedCmpNode(Node, CmpNode):
9139 9140 9141
    #  A CascadedCmpNode is not a complete expression node. It
    #  hangs off the side of another comparison node, shares
    #  its left operand with that node, and shares its result
William Stein's avatar
William Stein committed
9142 9143 9144 9145 9146 9147
    #  with the PrimaryCmpNode at the head of the chain.
    #
    #  operator      string
    #  operand2      ExprNode
    #  cascade       CascadedCmpNode

9148
    child_attrs = ['operand2', 'coerced_operand2', 'cascade']
Robert Bradshaw's avatar
Robert Bradshaw committed
9149

William Stein's avatar
William Stein committed
9150
    cascade = None
9151
    coerced_operand2 = None
9152 9153
    constant_result = constant_value_not_set # FIXME: where to calculate this?

Robert Bradshaw's avatar
Robert Bradshaw committed
9154 9155 9156 9157 9158 9159 9160
    def infer_type(self, env):
        # TODO: Actually implement this (after merging with -unstable).
        return py_object_type

    def type_dependencies(self, env):
        return ()

9161 9162 9163 9164
    def has_constant_result(self):
        return self.constant_result is not constant_value_not_set and \
               self.constant_result is not not_a_constant

9165
    def analyse_types(self, env):
William Stein's avatar
William Stein committed
9166 9167
        self.operand2.analyse_types(env)
        if self.cascade:
9168
            self.cascade.analyse_types(env)
9169

William Stein's avatar
William Stein committed
9170 9171
    def has_python_operands(self):
        return self.operand2.type.is_pyobject
9172

9173 9174 9175 9176
    def optimise_comparison(self, operand1, env):
        if self.find_special_bool_compare_function(env, operand1):
            if not operand1.type.is_pyobject:
                operand1 = operand1.coerce_to_pyobject(env)
9177
        if self.cascade:
9178 9179 9180
            operand2 = self.cascade.optimise_comparison(self.operand2, env)
            if operand2 is not self.operand2:
                self.coerced_operand2 = operand2
9181
        return operand1
9182

William Stein's avatar
William Stein committed
9183 9184
    def coerce_operands_to_pyobjects(self, env):
        self.operand2 = self.operand2.coerce_to_pyobject(env)
9185 9186
        if self.operand2.type is dict_type and self.operator in ('in', 'not_in'):
            self.operand2 = self.operand2.as_none_safe_node("'NoneType' object is not iterable")
William Stein's avatar
William Stein committed
9187 9188 9189 9190 9191 9192 9193 9194
        if self.cascade:
            self.cascade.coerce_operands_to_pyobjects(env)

    def coerce_cascaded_operands_to_temp(self, env):
        if self.cascade:
            #self.operand2 = self.operand2.coerce_to_temp(env) #CTT
            self.operand2 = self.operand2.coerce_to_simple(env)
            self.cascade.coerce_cascaded_operands_to_temp(env)
9195

9196
    def generate_evaluation_code(self, code, result, operand1, needs_evaluation=False):
9197 9198
        if self.type.is_pyobject:
            code.putln("if (__Pyx_PyObject_IsTrue(%s)) {" % result)
9199
            code.put_decref(result, self.type)
9200 9201
        else:
            code.putln("if (%s) {" % result)
9202 9203
        if needs_evaluation:
            operand1.generate_evaluation_code(code)
William Stein's avatar
William Stein committed
9204
        self.operand2.generate_evaluation_code(code)
9205
        self.generate_operation_code(code, result,
William Stein's avatar
William Stein committed
9206 9207 9208
            operand1, self.operator, self.operand2)
        if self.cascade:
            self.cascade.generate_evaluation_code(
9209 9210 9211 9212 9213
                code, result, self.coerced_operand2 or self.operand2,
                needs_evaluation=self.coerced_operand2 is not None)
        if needs_evaluation:
            operand1.generate_disposal_code(code)
            operand1.free_temps(code)
William Stein's avatar
William Stein committed
9214 9215
        # Cascaded cmp result is always temp
        self.operand2.generate_disposal_code(code)
9216
        self.operand2.free_temps(code)
William Stein's avatar
William Stein committed
9217 9218
        code.putln("}")

9219 9220 9221 9222 9223
    def annotate(self, code):
        self.operand2.annotate(code)
        if self.cascade:
            self.cascade.annotate(code)

William Stein's avatar
William Stein committed
9224 9225

binop_node_classes = {
9226 9227
    "or":       BoolBinopNode,
    "and":      BoolBinopNode,
Robert Bradshaw's avatar
Robert Bradshaw committed
9228 9229 9230
    "|":        IntBinopNode,
    "^":        IntBinopNode,
    "&":        IntBinopNode,
9231 9232
    "<<":       IntBinopNode,
    ">>":       IntBinopNode,
Robert Bradshaw's avatar
Robert Bradshaw committed
9233 9234 9235
    "+":        AddNode,
    "-":        SubNode,
    "*":        MulNode,
9236 9237
    "/":        DivNode,
    "//":       DivNode,
Robert Bradshaw's avatar
Robert Bradshaw committed
9238
    "%":        ModNode,
9239
    "**":       PowNode
William Stein's avatar
William Stein committed
9240 9241
}

9242
def binop_node(pos, operator, operand1, operand2, inplace=False):
9243
    # Construct binop node of appropriate class for
William Stein's avatar
William Stein committed
9244
    # given operator.
9245 9246 9247
    return binop_node_classes[operator](pos,
        operator = operator,
        operand1 = operand1,
9248 9249
        operand2 = operand2,
        inplace = inplace)
William Stein's avatar
William Stein committed
9250 9251 9252 9253 9254 9255 9256 9257 9258 9259 9260 9261

#-------------------------------------------------------------------
#
#  Coercion nodes
#
#  Coercion nodes are special in that they are created during
#  the analyse_types phase of parse tree processing.
#  Their __init__ methods consequently incorporate some aspects
#  of that phase.
#
#-------------------------------------------------------------------

9262
class CoercionNode(ExprNode):
William Stein's avatar
William Stein committed
9263 9264 9265
    #  Abstract base class for coercion nodes.
    #
    #  arg       ExprNode       node being coerced
9266

William Stein's avatar
William Stein committed
9267
    subexprs = ['arg']
9268
    constant_result = not_a_constant
9269

William Stein's avatar
William Stein committed
9270 9271 9272 9273
    def __init__(self, arg):
        self.pos = arg.pos
        self.arg = arg
        if debug_coercion:
Stefan Behnel's avatar
Stefan Behnel committed
9274
            print("%s Coercing %s" % (self, self.arg))
9275 9276

    def calculate_constant_result(self):
9277 9278
        # constant folding can break type coercion, so this is disabled
        pass
9279

9280 9281 9282 9283 9284
    def annotate(self, code):
        self.arg.annotate(code)
        if self.arg.type != self.type:
            file, line, col = self.pos
            code.annotate((file, line, col-1), AnnotationItem(style='coerce', tag='coerce', text='[%s] to [%s]' % (self.arg.type, self.type)))
William Stein's avatar
William Stein committed
9285

9286
class CoerceToMemViewSliceNode(CoercionNode):
9287 9288 9289 9290
    """
    Coerce an object to a memoryview slice. This holds a new reference in
    a managed temp.
    """
9291 9292

    def __init__(self, arg, dst_type, env):
9293 9294
        assert dst_type.is_memoryviewslice
        assert not arg.type.is_memoryviewslice
9295 9296
        CoercionNode.__init__(self, arg)
        self.type = dst_type
9297
        self.is_temp = 1
9298 9299
        self.env = env
        self.use_managed_ref = True
9300
        self.arg = arg
9301 9302

    def generate_result_code(self, code):
9303 9304 9305 9306 9307
        self.type.create_from_py_utility_code(self.env)
        code.putln("%s = %s(%s);" % (self.result(),
                                     self.type.from_py_function,
                                     self.arg.py_result()))

9308 9309 9310
        error_cond = self.type.error_condition(self.result())
        code.putln(code.error_goto_if(error_cond, self.pos))

William Stein's avatar
William Stein committed
9311 9312 9313

class CastNode(CoercionNode):
    #  Wrap a node in a C type cast.
9314

William Stein's avatar
William Stein committed
9315 9316 9317
    def __init__(self, arg, new_type):
        CoercionNode.__init__(self, arg)
        self.type = new_type
Stefan Behnel's avatar
Stefan Behnel committed
9318 9319 9320

    def may_be_none(self):
        return self.arg.may_be_none()
9321

William Stein's avatar
William Stein committed
9322 9323 9324 9325 9326 9327 9328 9329 9330 9331 9332 9333
    def calculate_result_code(self):
        return self.arg.result_as(self.type)

    def generate_result_code(self, code):
        self.arg.generate_result_code(code)


class PyTypeTestNode(CoercionNode):
    #  This node is used to check that a generic Python
    #  object is an instance of a particular extension type.
    #  This node borrows the result of its argument node.

9334
    def __init__(self, arg, dst_type, env, notnone=False):
William Stein's avatar
William Stein committed
9335 9336
        #  The arg is know to be a Python object, and
        #  the dst_type is known to be an extension type.
Robert Bradshaw's avatar
Robert Bradshaw committed
9337
        assert dst_type.is_extension_type or dst_type.is_builtin_type, "PyTypeTest on non extension type"
William Stein's avatar
William Stein committed
9338 9339 9340
        CoercionNode.__init__(self, arg)
        self.type = dst_type
        self.result_ctype = arg.ctype()
9341
        self.notnone = notnone
9342

9343
    nogil_check = Node.gil_error
9344
    gil_message = "Python type test"
9345

9346 9347
    def analyse_types(self, env):
        pass
Stefan Behnel's avatar
Stefan Behnel committed
9348 9349 9350 9351 9352

    def may_be_none(self):
        if self.notnone:
            return False
        return self.arg.may_be_none()
9353

9354 9355 9356
    def is_simple(self):
        return self.arg.is_simple()

William Stein's avatar
William Stein committed
9357 9358
    def result_in_temp(self):
        return self.arg.result_in_temp()
9359

William Stein's avatar
William Stein committed
9360 9361
    def is_ephemeral(self):
        return self.arg.is_ephemeral()
9362

9363
    def nonlocally_immutable(self):
9364
        return self.arg.nonlocally_immutable()
9365

9366 9367 9368 9369
    def calculate_constant_result(self):
        # FIXME
        pass

William Stein's avatar
William Stein committed
9370
    def calculate_result_code(self):
9371
        return self.arg.result()
9372

William Stein's avatar
William Stein committed
9373 9374
    def generate_result_code(self, code):
        if self.type.typeobj_is_available():
9375
            if not self.type.is_builtin_type:
9376
                code.globalstate.use_utility_code(UtilityCode.load_cached("ExtTypeTest", "ObjectHandling.c"))
William Stein's avatar
William Stein committed
9377
            code.putln(
Robert Bradshaw's avatar
Robert Bradshaw committed
9378
                "if (!(%s)) %s" % (
9379
                    self.type.type_test_code(self.arg.py_result(), self.notnone),
William Stein's avatar
William Stein committed
9380 9381 9382 9383
                    code.error_goto(self.pos)))
        else:
            error(self.pos, "Cannot test type of extern C class "
                "without type object name specification")
9384

William Stein's avatar
William Stein committed
9385 9386
    def generate_post_assignment_code(self, code):
        self.arg.generate_post_assignment_code(code)
9387 9388 9389

    def free_temps(self, code):
        self.arg.free_temps(code)
9390 9391 9392 9393 9394 9395 9396


class NoneCheckNode(CoercionNode):
    # This node is used to check that a Python object is not None and
    # raises an appropriate exception (as specified by the creating
    # transform).

9397 9398
    is_nonecheck = True

9399 9400
    def __init__(self, arg, exception_type_cname, exception_message,
                 exception_format_args):
9401 9402 9403 9404 9405
        CoercionNode.__init__(self, arg)
        self.type = arg.type
        self.result_ctype = arg.ctype()
        self.exception_type_cname = exception_type_cname
        self.exception_message = exception_message
9406
        self.exception_format_args = tuple(exception_format_args or ())
9407

9408 9409
    nogil_check = None # this node only guards an operation that would fail already

9410 9411 9412
    def analyse_types(self, env):
        pass

9413 9414 9415
    def may_be_none(self):
        return False

9416 9417 9418
    def is_simple(self):
        return self.arg.is_simple()

9419 9420 9421
    def result_in_temp(self):
        return self.arg.result_in_temp()

9422
    def nonlocally_immutable(self):
9423
        return self.arg.nonlocally_immutable()
9424

9425 9426
    def calculate_result_code(self):
        return self.arg.result()
9427

9428 9429 9430 9431
    def condition(self):
        if self.type.is_pyobject:
            return self.arg.py_result()
        elif self.type.is_memoryviewslice:
9432
            return "((PyObject *) %s.memview)" % self.arg.result()
9433 9434 9435 9436
        else:
            raise Exception("unsupported type")

    def put_nonecheck(self, code):
9437
        code.putln(
9438 9439 9440 9441 9442
            "if (unlikely(%s == Py_None)) {" % self.condition())

        if self.in_nogil_context:
            code.put_ensure_gil()

9443 9444
        escape = StringEncoding.escape_byte_string
        if self.exception_format_args:
9445
            code.putln('PyErr_Format(%s, "%s", %s);' % (
9446 9447 9448 9449
                self.exception_type_cname,
                StringEncoding.escape_byte_string(
                    self.exception_message.encode('UTF-8')),
                ', '.join([ '"%s"' % escape(str(arg).encode('UTF-8'))
9450
                            for arg in self.exception_format_args ])))
9451
        else:
9452
            code.putln('PyErr_SetString(%s, "%s");' % (
9453
                self.exception_type_cname,
9454 9455 9456 9457 9458 9459
                escape(self.exception_message.encode('UTF-8'))))

        if self.in_nogil_context:
            code.put_release_ensured_gil()

        code.putln(code.error_goto(self.pos))
9460 9461
        code.putln("}")

9462 9463 9464
    def generate_result_code(self, code):
        self.put_nonecheck(code)

9465 9466 9467 9468 9469 9470
    def generate_post_assignment_code(self, code):
        self.arg.generate_post_assignment_code(code)

    def free_temps(self, code):
        self.arg.free_temps(code)

9471

William Stein's avatar
William Stein committed
9472 9473 9474
class CoerceToPyTypeNode(CoercionNode):
    #  This node is used to convert a C data type
    #  to a Python object.
9475

9476
    type = py_object_type
Robert Bradshaw's avatar
Robert Bradshaw committed
9477
    is_temp = 1
William Stein's avatar
William Stein committed
9478

9479
    def __init__(self, arg, env, type=py_object_type):
9480
        if not arg.type.create_to_py_utility_code(env):
9481 9482 9483 9484 9485 9486 9487
            error(arg.pos, "Cannot convert '%s' to Python object" % arg.type)
        elif arg.type.is_complex:
            # special case: complex coercion is so complex that it
            # uses a macro ("__pyx_PyComplex_FromComplex()"), for
            # which the argument must be simple
            arg = arg.coerce_to_simple(env)
        CoercionNode.__init__(self, arg)
9488 9489
        if type is py_object_type:
            # be specific about some known types
9490
            if arg.type.is_string or arg.type.is_cpp_string:
9491
                self.type = bytes_type
Stefan Behnel's avatar
Stefan Behnel committed
9492
            elif arg.type.is_unicode_char:
9493 9494 9495 9496 9497 9498
                self.type = unicode_type
            elif arg.type.is_complex:
                self.type = Builtin.complex_type
        else:
            # FIXME: check that the target type and the resulting type are compatible
            pass
9499

9500 9501 9502 9503
        if arg.type.is_memoryviewslice:
            # Register utility codes at this point
            arg.type.get_to_py_function(env, arg)

9504 9505
        self.env = env

9506
    gil_message = "Converting to Python object"
9507

9508 9509 9510 9511
    def may_be_none(self):
        # FIXME: is this always safe?
        return False

9512
    def coerce_to_boolean(self, env):
9513 9514 9515 9516 9517 9518
        arg_type = self.arg.type
        if (arg_type == PyrexTypes.c_bint_type or
            (arg_type.is_pyobject and arg_type.name == 'bool')):
            return self.arg.coerce_to_temp(env)
        else:
            return CoerceToBooleanNode(self, env)
9519

9520 9521 9522 9523 9524 9525
    def coerce_to_integer(self, env):
        # If not already some C integer type, coerce to longint.
        if self.arg.type.is_int:
            return self.arg
        else:
            return self.arg.coerce_to(PyrexTypes.c_long_type, env)
9526

9527 9528 9529 9530
    def analyse_types(self, env):
        # The arg is always already analysed
        pass

William Stein's avatar
William Stein committed
9531
    def generate_result_code(self, code):
9532
        if self.arg.type.is_memoryviewslice:
9533
            funccall = self.arg.type.get_to_py_function(self.env, self.arg)
9534 9535 9536 9537 9538
        else:
            funccall = "%s(%s)" % (self.arg.type.to_py_function,
                                   self.arg.result())

        code.putln('%s = %s; %s' % (
9539
            self.result(),
9540
            funccall,
9541
            code.error_goto_if_null(self.result(), self.pos)))
9542

9543
        code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
9544 9545


9546 9547 9548 9549 9550 9551 9552 9553 9554 9555 9556 9557 9558 9559 9560 9561 9562 9563 9564 9565 9566 9567 9568 9569 9570 9571 9572 9573 9574 9575 9576 9577 9578 9579 9580 9581 9582 9583 9584 9585
class CoerceIntToBytesNode(CoerceToPyTypeNode):
    #  This node is used to convert a C int type to a Python bytes
    #  object.

    is_temp = 1

    def __init__(self, arg, env):
        arg = arg.coerce_to_simple(env)
        CoercionNode.__init__(self, arg)
        self.type = Builtin.bytes_type

    def generate_result_code(self, code):
        arg = self.arg
        arg_result = arg.result()
        if arg.type not in (PyrexTypes.c_char_type,
                            PyrexTypes.c_uchar_type,
                            PyrexTypes.c_schar_type):
            if arg.type.signed:
                code.putln("if ((%s < 0) || (%s > 255)) {" % (
                    arg_result, arg_result))
            else:
                code.putln("if (%s > 255) {" % arg_result)
            code.putln('PyErr_Format(PyExc_OverflowError, '
                       '"value too large to pack into a byte"); %s' % (
                           code.error_goto(self.pos)))
            code.putln('}')
        temp = None
        if arg.type is not PyrexTypes.c_char_type:
            temp = code.funcstate.allocate_temp(PyrexTypes.c_char_type, manage_ref=False)
            code.putln("%s = (char)%s;" % (temp, arg_result))
            arg_result = temp
        code.putln('%s = PyBytes_FromStringAndSize(&%s, 1); %s' % (
            self.result(),
            arg_result,
            code.error_goto_if_null(self.result(), self.pos)))
        if temp is not None:
            code.funcstate.release_temp(temp)
        code.put_gotref(self.py_result())


William Stein's avatar
William Stein committed
9586 9587 9588 9589 9590 9591 9592 9593
class CoerceFromPyTypeNode(CoercionNode):
    #  This node is used to convert a Python object
    #  to a C data type.

    def __init__(self, result_type, arg, env):
        CoercionNode.__init__(self, arg)
        self.type = result_type
        self.is_temp = 1
9594
        if not result_type.create_from_py_utility_code(env):
William Stein's avatar
William Stein committed
9595
            error(arg.pos,
Craig Citro's avatar
Craig Citro committed
9596
                  "Cannot convert Python object to '%s'" % result_type)
9597 9598 9599 9600 9601 9602 9603 9604
        if self.type.is_string:
            if self.arg.is_ephemeral():
                error(arg.pos,
                      "Obtaining char* from temporary Python value")
            elif self.arg.is_name and self.arg.entry and self.arg.entry.is_pyglobal:
                warning(arg.pos,
                        "Obtaining char* from externally modifiable global Python value",
                        level=1)
9605

9606 9607 9608 9609
    def analyse_types(self, env):
        # The arg is always already analysed
        pass

William Stein's avatar
William Stein committed
9610 9611
    def generate_result_code(self, code):
        function = self.type.from_py_function
9612 9613 9614 9615
        operand = self.arg.py_result()
        rhs = "%s(%s)" % (function, operand)
        if self.type.is_enum:
            rhs = typecast(self.type, c_long_type, rhs)
Robert Bradshaw's avatar
Robert Bradshaw committed
9616
        code.putln('%s = %s; %s' % (
9617
            self.result(),
9618
            rhs,
9619
            code.error_goto_if(self.type.error_condition(self.result()), self.pos)))
9620
        if self.type.is_pyobject:
9621
            code.put_gotref(self.py_result())
William Stein's avatar
William Stein committed
9622

9623 9624 9625
    def nogil_check(self, env):
        error(self.pos, "Coercion from Python not allowed without the GIL")

William Stein's avatar
William Stein committed
9626 9627 9628 9629

class CoerceToBooleanNode(CoercionNode):
    #  This node is used when a result needs to be used
    #  in a boolean context.
9630

9631
    type = PyrexTypes.c_bint_type
9632 9633 9634 9635

    _special_builtins = {
        Builtin.list_type    : 'PyList_GET_SIZE',
        Builtin.tuple_type   : 'PyTuple_GET_SIZE',
9636
        Builtin.bytes_type   : 'PyBytes_GET_SIZE',
9637 9638 9639
        Builtin.unicode_type : 'PyUnicode_GET_SIZE',
        }

William Stein's avatar
William Stein committed
9640 9641 9642 9643
    def __init__(self, arg, env):
        CoercionNode.__init__(self, arg)
        if arg.type.is_pyobject:
            self.is_temp = 1
9644

9645
    def nogil_check(self, env):
9646
        if self.arg.type.is_pyobject and self._special_builtins.get(self.arg.type) is None:
9647
            self.gil_error()
9648

9649
    gil_message = "Truth-testing Python object"
9650

William Stein's avatar
William Stein committed
9651 9652 9653
    def check_const(self):
        if self.is_temp:
            self.not_const()
9654 9655
            return False
        return self.arg.check_const()
9656

William Stein's avatar
William Stein committed
9657
    def calculate_result_code(self):
9658
        return "(%s != 0)" % self.arg.result()
William Stein's avatar
William Stein committed
9659 9660

    def generate_result_code(self, code):
9661 9662 9663 9664
        if not self.is_temp:
            return
        test_func = self._special_builtins.get(self.arg.type)
        if test_func is not None:
Stefan Behnel's avatar
Stefan Behnel committed
9665
            code.putln("%s = (%s != Py_None) && (%s(%s) != 0);" % (
9666 9667 9668 9669 9670
                       self.result(),
                       self.arg.py_result(),
                       test_func,
                       self.arg.py_result()))
        else:
William Stein's avatar
William Stein committed
9671
            code.putln(
9672
                "%s = __Pyx_PyObject_IsTrue(%s); %s" % (
9673 9674
                    self.result(),
                    self.arg.py_result(),
9675
                    code.error_goto_if_neg(self.result(), self.pos)))
William Stein's avatar
William Stein committed
9676

9677 9678 9679 9680 9681 9682 9683 9684 9685 9686 9687
class CoerceToComplexNode(CoercionNode):

    def __init__(self, arg, dst_type, env):
        if arg.type.is_complex:
            arg = arg.coerce_to_simple(env)
        self.type = dst_type
        CoercionNode.__init__(self, arg)
        dst_type.create_declaration_utility_code(env)

    def calculate_result_code(self):
        if self.arg.type.is_complex:
9688 9689
            real_part = "__Pyx_CREAL(%s)" % self.arg.result()
            imag_part = "__Pyx_CIMAG(%s)" % self.arg.result()
9690 9691 9692 9693 9694 9695 9696
        else:
            real_part = self.arg.result()
            imag_part = "0"
        return "%s(%s, %s)" % (
                self.type.from_parts,
                real_part,
                imag_part)
9697

9698 9699
    def generate_result_code(self, code):
        pass
William Stein's avatar
William Stein committed
9700 9701 9702 9703 9704 9705 9706 9707

class CoerceToTempNode(CoercionNode):
    #  This node is used to force the result of another node
    #  to be stored in a temporary. It is only used if the
    #  argument node's result is not already in a temporary.

    def __init__(self, arg, env):
        CoercionNode.__init__(self, arg)
9708
        self.type = self.arg.type.as_argument_type()
9709
        self.constant_result = self.arg.constant_result
William Stein's avatar
William Stein committed
9710 9711 9712
        self.is_temp = 1
        if self.type.is_pyobject:
            self.result_ctype = py_object_type
9713 9714 9715

    gil_message = "Creating temporary Python reference"

9716 9717 9718
    def analyse_types(self, env):
        # The arg is always already analysed
        pass
9719

9720 9721
    def coerce_to_boolean(self, env):
        self.arg = self.arg.coerce_to_boolean(env)
9722 9723
        if self.arg.is_simple():
            return self.arg
9724 9725 9726
        self.type = self.arg.type
        self.result_ctype = self.type
        return self
9727

William Stein's avatar
William Stein committed
9728 9729 9730 9731
    def generate_result_code(self, code):
        #self.arg.generate_evaluation_code(code) # Already done
        # by generic generate_subexpr_evaluation_code!
        code.putln("%s = %s;" % (
9732
            self.result(), self.arg.result_as(self.ctype())))
9733 9734 9735 9736 9737 9738
        if self.use_managed_ref:
            if self.type.is_pyobject:
                code.put_incref(self.result(), self.ctype())
            elif self.type.is_memoryviewslice:
                code.put_incref_memoryviewslice(self.result(),
                                                not self.in_nogil_context)
William Stein's avatar
William Stein committed
9739

9740 9741 9742 9743 9744 9745 9746 9747 9748
class ProxyNode(CoercionNode):
    """
    A node that should not be replaced by transforms or other means,
    and hence can be useful to wrap the argument to a clone node

    MyNode    -> ProxyNode -> ArgNode
    CloneNode -^
    """

9749 9750
    nogil_check = None

9751 9752
    def __init__(self, arg):
        super(ProxyNode, self).__init__(arg)
9753 9754 9755 9756 9757 9758 9759 9760 9761 9762 9763 9764
        self._proxy_type()

    def analyse_expressions(self, env):
        self.arg.analyse_expressions(env)
        self._proxy_type()

    def _proxy_type(self):
        if hasattr(self.arg, 'type'):
            self.type = self.arg.type
            self.result_ctype = self.arg.result_ctype
        if hasattr(self.arg, 'entry'):
            self.entry = self.arg.entry
9765 9766 9767 9768 9769 9770 9771 9772 9773 9774 9775 9776 9777 9778 9779 9780 9781 9782 9783 9784 9785 9786 9787 9788

    def generate_result_code(self, code):
        self.arg.generate_result_code(code)

    def result(self):
        return self.arg.result()

    def is_simple(self):
        return self.arg.is_simple()

    def may_be_none(self):
        return self.arg.may_be_none()

    def generate_evaluation_code(self, code):
        self.arg.generate_evaluation_code(code)

    def generate_result_code(self, code):
        self.arg.generate_result_code(code)

    def generate_disposal_code(self, code):
        self.arg.generate_disposal_code(code)

    def free_temps(self, code):
        self.arg.free_temps(code)
William Stein's avatar
William Stein committed
9789 9790 9791 9792 9793 9794

class CloneNode(CoercionNode):
    #  This node is employed when the result of another node needs
    #  to be used multiple times. The argument node's result must
    #  be in a temporary. This node "borrows" the result from the
    #  argument node, and does not generate any evaluation or
9795
    #  disposal code for it. The original owner of the argument
William Stein's avatar
William Stein committed
9796
    #  node is responsible for doing those things.
9797

William Stein's avatar
William Stein committed
9798
    subexprs = [] # Arg is not considered a subexpr
9799
    nogil_check = None
9800

William Stein's avatar
William Stein committed
9801 9802
    def __init__(self, arg):
        CoercionNode.__init__(self, arg)
9803 9804 9805 9806 9807
        if hasattr(arg, 'type'):
            self.type = arg.type
            self.result_ctype = arg.result_ctype
        if hasattr(arg, 'entry'):
            self.entry = arg.entry
9808

9809
    def result(self):
9810
        return self.arg.result()
9811

9812 9813 9814
    def may_be_none(self):
        return self.arg.may_be_none()

Robert Bradshaw's avatar
Robert Bradshaw committed
9815 9816
    def type_dependencies(self, env):
        return self.arg.type_dependencies(env)
9817

9818 9819
    def infer_type(self, env):
        return self.arg.infer_type(env)
Robert Bradshaw's avatar
Robert Bradshaw committed
9820

Robert Bradshaw's avatar
Robert Bradshaw committed
9821 9822 9823 9824
    def analyse_types(self, env):
        self.type = self.arg.type
        self.result_ctype = self.arg.result_ctype
        self.is_temp = 1
9825 9826
        if hasattr(self.arg, 'entry'):
            self.entry = self.arg.entry
9827

9828 9829 9830
    def is_simple(self):
        return True # result is always in a temp (or a name)

William Stein's avatar
William Stein committed
9831 9832 9833 9834 9835
    def generate_evaluation_code(self, code):
        pass

    def generate_result_code(self, code):
        pass
9836

9837
    def generate_disposal_code(self, code):
9838
        pass
9839

9840 9841
    def free_temps(self, code):
        pass
9842

9843

Stefan Behnel's avatar
Stefan Behnel committed
9844 9845 9846 9847 9848 9849 9850 9851 9852 9853 9854 9855
class CMethodSelfCloneNode(CloneNode):
    # Special CloneNode for the self argument of builtin C methods
    # that accepts subtypes of the builtin type.  This is safe only
    # for 'final' subtypes, as subtypes of the declared type may
    # override the C method.

    def coerce_to(self, dst_type, env):
        if dst_type.is_builtin_type and self.type.subtype_of(dst_type):
            return self
        return CloneNode.coerce_to(self, dst_type, env)


9856 9857
class ModuleRefNode(ExprNode):
    # Simple returns the module object
9858

9859 9860 9861
    type = py_object_type
    is_temp = False
    subexprs = []
9862

9863 9864 9865
    def analyse_types(self, env):
        pass

9866 9867 9868
    def may_be_none(self):
        return False

9869 9870 9871 9872 9873 9874 9875 9876
    def calculate_result_code(self):
        return Naming.module_cname

    def generate_result_code(self, code):
        pass

class DocstringRefNode(ExprNode):
    # Extracts the docstring of the body element
9877

9878 9879 9880
    subexprs = ['body']
    type = py_object_type
    is_temp = True
9881

9882 9883 9884 9885 9886 9887 9888 9889 9890
    def __init__(self, pos, body):
        ExprNode.__init__(self, pos)
        assert body.type.is_pyobject
        self.body = body

    def analyse_types(self, env):
        pass

    def generate_result_code(self, code):
9891 9892 9893
        code.putln('%s = __Pyx_GetAttrString(%s, "__doc__"); %s' % (
            self.result(), self.body.result(),
            code.error_goto_if_null(self.result(), self.pos)))
9894 9895 9896 9897
        code.put_gotref(self.result())



William Stein's avatar
William Stein committed
9898 9899 9900 9901 9902 9903
#------------------------------------------------------------------------------------
#
#  Runtime support code
#
#------------------------------------------------------------------------------------

9904 9905
get_name_interned_utility_code = UtilityCode(
proto = """
9906
static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name); /*proto*/
9907 9908
""",
impl = """
William Stein's avatar
William Stein committed
9909 9910 9911
static PyObject *__Pyx_GetName(PyObject *dict, PyObject *name) {
    PyObject *result;
    result = PyObject_GetAttr(dict, name);
9912 9913 9914 9915 9916 9917 9918 9919 9920
    if (!result) {
        if (dict != %(BUILTINS)s) {
            PyErr_Clear();
            result = PyObject_GetAttr(%(BUILTINS)s, name);
        }
        if (!result) {
            PyErr_SetObject(PyExc_NameError, name);
        }
    }
William Stein's avatar
William Stein committed
9921 9922
    return result;
}
9923
""" % {'BUILTINS' : Naming.builtins_cname})
William Stein's avatar
William Stein committed
9924 9925 9926

#------------------------------------------------------------------------------------

9927 9928 9929 9930 9931 9932 9933 9934 9935 9936 9937 9938 9939 9940 9941 9942 9943 9944 9945
pyerr_occurred_withgil_utility_code= UtilityCode(
proto = """
static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void); /* proto */
""",
impl = """
static CYTHON_INLINE int __Pyx_ErrOccurredWithGIL(void) {
  int err;
  #ifdef WITH_THREAD
  PyGILState_STATE _save = PyGILState_Ensure();
  #endif
  err = !!PyErr_Occurred();
  #ifdef WITH_THREAD
  PyGILState_Release(_save);
  #endif
  return err;
}
"""
)

Robert Bradshaw's avatar
Robert Bradshaw committed
9946
#------------------------------------------------------------------------------------
Robert Bradshaw's avatar
Robert Bradshaw committed
9947

9948 9949 9950 9951 9952 9953 9954 9955 9956 9957 9958 9959 9960 9961 9962 9963 9964 9965 9966 9967
raise_unbound_local_error_utility_code = UtilityCode(
proto = """
static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname);
""",
impl = """
static CYTHON_INLINE void __Pyx_RaiseUnboundLocalError(const char *varname) {
    PyErr_Format(PyExc_UnboundLocalError, "local variable '%s' referenced before assignment", varname);
}
""")

raise_closure_name_error_utility_code = UtilityCode(
proto = """
static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname);
""",
impl = """
static CYTHON_INLINE void __Pyx_RaiseClosureNameError(const char *varname) {
    PyErr_Format(PyExc_NameError, "free variable '%s' referenced before assignment in enclosing scope", varname);
}
""")

9968 9969 9970 9971 9972 9973 9974 9975 9976 9977 9978
# Don't inline the function, it should really never be called in production
raise_unbound_memoryview_utility_code_nogil = UtilityCode(
proto = """
static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname);
""",
impl = """
static void __Pyx_RaiseUnboundMemoryviewSliceNogil(const char *varname) {
    #ifdef WITH_THREAD
    PyGILState_STATE gilstate = PyGILState_Ensure();
    #endif
    __Pyx_RaiseUnboundLocalError(varname);
9979
    #ifdef WITH_THREAD
9980 9981 9982 9983 9984 9985
    PyGILState_Release(gilstate);
    #endif
}
""",
requires = [raise_unbound_local_error_utility_code])

9986 9987
#------------------------------------------------------------------------------------

9988 9989 9990
raise_too_many_values_to_unpack = UtilityCode.load_cached("RaiseTooManyValuesToUnpack", "ObjectHandling.c")
raise_need_more_values_to_unpack = UtilityCode.load_cached("RaiseNeedMoreValuesToUnpack", "ObjectHandling.c")
tuple_unpacking_error_code = UtilityCode.load_cached("UnpackTupleError", "ObjectHandling.c")
Robert Bradshaw's avatar
Robert Bradshaw committed
9991 9992 9993 9994 9995

#------------------------------------------------------------------------------------

int_pow_utility_code = UtilityCode(
proto="""
9996
static CYTHON_INLINE %(type)s %(func_name)s(%(type)s, %(type)s); /* proto */
Robert Bradshaw's avatar
Robert Bradshaw committed
9997 9998
""",
impl="""
9999
static CYTHON_INLINE %(type)s %(func_name)s(%(type)s b, %(type)s e) {
Robert Bradshaw's avatar
Robert Bradshaw committed
10000 10001 10002 10003 10004 10005 10006 10007 10008 10009 10010 10011 10012 10013 10014 10015 10016 10017 10018 10019 10020
    %(type)s t = b;
    switch (e) {
        case 3:
            t *= b;
        case 2:
            t *= b;
        case 1:
            return t;
        case 0:
            return 1;
    }
    if (unlikely(e<0)) return 0;
    t = 1;
    while (likely(e)) {
        t *= (b * (e&1)) | ((~e)&1);    /* 1 or b */
        b *= b;
        e >>= 1;
    }
    return t;
}
""")
10021 10022 10023

# ------------------------------ Division ------------------------------------

10024 10025
div_int_utility_code = UtilityCode(
proto="""
10026
static CYTHON_INLINE %(type)s __Pyx_div_%(type_name)s(%(type)s, %(type)s); /* proto */
10027 10028
""",
impl="""
10029
static CYTHON_INLINE %(type)s __Pyx_div_%(type_name)s(%(type)s a, %(type)s b) {
10030 10031 10032 10033 10034
    %(type)s q = a / b;
    %(type)s r = a - q*b;
    q -= ((r != 0) & ((r ^ b) < 0));
    return q;
}
10035 10036
""")

10037
mod_int_utility_code = UtilityCode(
10038
proto="""
10039
static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s, %(type)s); /* proto */
10040 10041
""",
impl="""
10042
static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s a, %(type)s b) {
10043 10044 10045
    %(type)s r = a %% b;
    r += ((r != 0) & ((r ^ b) < 0)) * b;
    return r;
10046 10047 10048
}
""")

10049
mod_float_utility_code = UtilityCode(
10050
proto="""
10051
static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s, %(type)s); /* proto */
10052 10053
""",
impl="""
10054
static CYTHON_INLINE %(type)s __Pyx_mod_%(type_name)s(%(type)s a, %(type)s b) {
10055 10056 10057
    %(type)s r = fmod%(math_h_modifier)s(a, b);
    r += ((r != 0) & ((r < 0) ^ (b < 0))) * b;
    return r;
10058 10059
}
""")
Robert Bradshaw's avatar
Robert Bradshaw committed
10060

10061
cdivision_warning_utility_code = UtilityCode(
Robert Bradshaw's avatar
Robert Bradshaw committed
10062
proto="""
10063
static int __Pyx_cdivision_warning(const char *, int); /* proto */
Robert Bradshaw's avatar
Robert Bradshaw committed
10064 10065
""",
impl="""
10066
static int __Pyx_cdivision_warning(const char *filename, int lineno) {
10067 10068 10069 10070 10071 10072
#if CYTHON_COMPILING_IN_PYPY
    filename++; // avoid compiler warnings
    lineno++;
    return PyErr_Warn(PyExc_RuntimeWarning,
                     "division with oppositely signed operands, C and Python semantics differ");
#else
10073
    return PyErr_WarnExplicit(PyExc_RuntimeWarning,
10074
                              "division with oppositely signed operands, C and Python semantics differ",
10075 10076
                              filename,
                              lineno,
10077
                              __Pyx_MODULE_NAME,
10078
                              NULL);
10079
#endif
Robert Bradshaw's avatar
Robert Bradshaw committed
10080
}
10081
""")
10082 10083 10084 10085

# from intobject.c
division_overflow_test_code = UtilityCode(
proto="""
Vitja Makarov's avatar
Vitja Makarov committed
10086 10087
#define UNARY_NEG_WOULD_OVERFLOW(x)    \
        (((x) < 0) & ((unsigned long)(x) == 0-(unsigned long)(x)))
10088
""")