Code.py 92.7 KB
Newer Older
1
# cython: language_level = 2
William Stein's avatar
William Stein committed
2
#
3
#   Code output module
William Stein's avatar
William Stein committed
4 5
#

6 7
from __future__ import absolute_import

Stefan Behnel's avatar
Stefan Behnel committed
8
import cython
Stefan Behnel's avatar
Stefan Behnel committed
9 10
cython.declare(os=object, re=object, operator=object, textwrap=object,
               Template=object, Naming=object, Options=object, StringEncoding=object,
Stefan Behnel's avatar
Stefan Behnel committed
11
               Utils=object, SourceDescriptor=object, StringIOTree=object,
Stefan Behnel's avatar
Stefan Behnel committed
12 13
               DebugFlags=object, basestring=object, defaultdict=object,
               closing=object, partial=object)
Stefan Behnel's avatar
Stefan Behnel committed
14

15
import os
16
import re
17
import shutil
Stefan Behnel's avatar
Stefan Behnel committed
18
import sys
19
import operator
20
import textwrap
21 22
from string import Template
from functools import partial
Stefan Behnel's avatar
Stefan Behnel committed
23 24
from contextlib import closing
from collections import defaultdict
25

Robert Bradshaw's avatar
Robert Bradshaw committed
26 27 28 29 30
try:
    import hashlib
except ImportError:
    import md5 as hashlib

31 32 33 34
from . import Naming
from . import Options
from . import DebugFlags
from . import StringEncoding
35
from . import Version
36 37 38
from .. import Utils
from .Scanning import SourceDescriptor
from ..StringIOTree import StringIOTree
39

Stefan Behnel's avatar
Stefan Behnel committed
40
try:
Stefan Behnel's avatar
Stefan Behnel committed
41 42 43
    from __builtin__ import basestring
except ImportError:
    from builtins import str as basestring
44

Stefan Behnel's avatar
Stefan Behnel committed
45
KEYWORDS_MUST_BE_BYTES = sys.version_info < (2, 7)
Stefan Behnel's avatar
Stefan Behnel committed
46

47 48

non_portable_builtins_map = {
49
    # builtins that have different names in different Python versions
50 51
    'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
    'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
52
    'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
53
    'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
54
    'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
55
}
56

57 58 59 60 61 62 63 64 65 66
ctypedef_builtins_map = {
    # types of builtins in "ctypedef class" statements which we don't
    # import either because the names conflict with C types or because
    # the type simply is not exposed.
    'py_int'             : '&PyInt_Type',
    'py_long'            : '&PyLong_Type',
    'py_float'           : '&PyFloat_Type',
    'wrapper_descriptor' : '&PyWrapperDescr_Type',
}

67 68
basicsize_builtins_map = {
    # builtins whose type has a different tp_basicsize than sizeof(...)
69 70
    'PyTypeObject': 'PyHeapTypeObject',
}
71

72
uncachable_builtins = [
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105
    # Global/builtin names that cannot be cached because they may or may not
    # be available at import time, for various reasons:
    ## - Py3.7+
    'breakpoint',  # might deserve an implementation in Cython
    ## - Py3.4+
    '__loader__',
    '__spec__',
    ## - Py3+
    'BlockingIOError',
    'BrokenPipeError',
    'ChildProcessError',
    'ConnectionAbortedError',
    'ConnectionError',
    'ConnectionRefusedError',
    'ConnectionResetError',
    'FileExistsError',
    'FileNotFoundError',
    'InterruptedError',
    'IsADirectoryError',
    'ModuleNotFoundError',
    'NotADirectoryError',
    'PermissionError',
    'ProcessLookupError',
    'RecursionError',
    'ResourceWarning',
    #'StopAsyncIteration',  # backported
    'TimeoutError',
    '__build_class__',
    'ascii',  # might deserve an implementation in Cython
    #'exec',  # implemented in Cython
    ## - Py2.7+
    'memoryview',
    ## - platform specific
106
    'WindowsError',
107 108
    ## - others
    '_',  # e.g. used by gettext
109
]
110

111 112 113 114 115 116 117
special_py_methods = set([
    '__cinit__', '__dealloc__', '__richcmp__', '__next__',
    '__await__', '__aiter__', '__anext__',
    '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__',
    '__getcharbuffer__', '__getbuffer__', '__releasebuffer__'
])

118 119 120 121
modifier_output_mapper = {
    'inline': 'CYTHON_INLINE'
}.get

Stefan Behnel's avatar
Stefan Behnel committed
122

123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165
class IncludeCode(object):
    """
    An include file and/or verbatim C code to be included in the
    generated sources.
    """
    # attributes:
    #
    #  pieces    {order: unicode}: pieces of C code to be generated.
    #            For the included file, the key "order" is zero.
    #            For verbatim include code, the "order" is the "order"
    #            attribute of the original IncludeCode where this piece
    #            of C code was first added. This is needed to prevent
    #            duplication if the same include code is found through
    #            multiple cimports.
    #  location  int: where to put this include in the C sources, one
    #            of the constants INITIAL, EARLY, LATE
    #  order     int: sorting order (automatically set by increasing counter)

    # Constants for location. If the same include occurs with different
    # locations, the earliest one takes precedense.
    INITIAL = 0
    EARLY = 1
    LATE = 2

    counter = 1   # Counter for "order"

    def __init__(self, include=None, verbatim=None, late=True, initial=False):
        self.order = self.counter
        type(self).counter += 1
        self.pieces = {}

        if include:
            if include[0] == '<' and include[-1] == '>':
                self.pieces[0] = u'#include {0}'.format(include)
                late = False  # system include is never late
            else:
                self.pieces[0] = u'#include "{0}"'.format(include)

        if verbatim:
            self.pieces[self.order] = verbatim

        if initial:
            self.location = self.INITIAL
166
        elif late:
167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198
            self.location = self.LATE
        else:
            self.location = self.EARLY

    def dict_update(self, d, key):
        """
        Insert `self` in dict `d` with key `key`. If that key already
        exists, update the attributes of the existing value with `self`.
        """
        if key in d:
            other = d[key]
            other.location = min(self.location, other.location)
            other.pieces.update(self.pieces)
        else:
            d[key] = self

    def sortkey(self):
        return self.order

    def mainpiece(self):
        """
        Return the main piece of C code, corresponding to the include
        file. If there was no include file, return None.
        """
        return self.pieces.get(0)

    def write(self, code):
        # Write values of self.pieces dict, sorted by the keys
        for k in sorted(self.pieces):
            code.putln(self.pieces[k])


199 200 201 202 203
def get_utility_dir():
    # make this a function and not global variables:
    # http://trac.cython.org/cython_trac/ticket/475
    Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(Cython_dir, "Utility")
Mark Florisson's avatar
Mark Florisson committed
204

Stefan Behnel's avatar
Stefan Behnel committed
205

206
class UtilityCodeBase(object):
207 208 209 210 211 212
    """
    Support for loading utility code from a file.

    Code sections in the file can be specified as follows:

        ##### MyUtility.proto #####
213

214
        [proto declarations]
215

216
        ##### MyUtility.init #####
217

218
        [code run at module initialization]
219 220 221

        ##### MyUtility #####
        #@requires: MyOtherUtility
222
        #@substitute: naming
223

224
        [definitions]
225 226 227 228 229 230 231 232 233

    for prototypes and implementation respectively.  For non-python or
    -cython files backslashes should be used instead.  5 to 30 comment
    characters may be used on either side.

    If the @cname decorator is not used and this is a CythonUtilityCode,
    one should pass in the 'name' keyword argument to be used for name
    mangling of such entries.
    """
234 235 236 237

    is_cython_utility = False
    _utility_cache = {}

238
    @classmethod
239
    def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
240 241
        if utility is None:
            return
242

243 244 245 246 247
        code = '\n'.join(lines)
        if tags and 'substitute' in tags and tags['substitute'] == set(['naming']):
            del tags['substitute']
            try:
                code = Template(code).substitute(vars(Naming))
248
            except (KeyError, ValueError) as e:
249 250 251 252 253
                raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % (
                    type, begin_lineno, e))

        # remember correct line numbers at least until after templating
        code = '\n' * begin_lineno + code
254

255
        if type == 'proto':
256
            utility[0] = code
257
        elif type == 'impl':
258
            utility[1] = code
259
        else:
260
            all_tags = utility[2]
Stefan Behnel's avatar
Stefan Behnel committed
261 262
            if KEYWORDS_MUST_BE_BYTES:
                type = type.encode('ASCII')
263
            all_tags[type] = code
264

265
        if tags:
266
            all_tags = utility[2]
Stefan Behnel's avatar
Stefan Behnel committed
267
            for name, values in tags.items():
Stefan Behnel's avatar
Stefan Behnel committed
268 269
                if KEYWORDS_MUST_BE_BYTES:
                    name = name.encode('ASCII')
270
                all_tags.setdefault(name, set()).update(values)
271

272
    @classmethod
273 274 275 276 277
    def load_utilities_from_file(cls, path):
        utilities = cls._utility_cache.get(path)
        if utilities:
            return utilities

278
        filename = os.path.join(get_utility_dir(), path)
279 280 281
        _, ext = os.path.splitext(path)
        if ext in ('.pyx', '.py', '.pxd', '.pxi'):
            comment = '#'
Stefan Behnel's avatar
Stefan Behnel committed
282
            strip_comments = partial(re.compile(r'^\s*#.*').sub, '')
Stefan Behnel's avatar
Stefan Behnel committed
283
            rstrip = StringEncoding._unicode.rstrip
284
        else:
285
            comment = '/'
Stefan Behnel's avatar
Stefan Behnel committed
286
            strip_comments = partial(re.compile(r'^\s*//.*|/\*[^*]*\*/').sub, '')
287
            rstrip = partial(re.compile(r'\s+(\\?)$').sub, r'\1')
288
        match_special = re.compile(
289
            (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'
Stefan Behnel's avatar
Stefan Behnel committed
290 291
             r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)') %
            {'C': comment}).match
292
        match_type = re.compile('(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match
293

Stefan Behnel's avatar
Stefan Behnel committed
294
        with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f:
295
            all_lines = f.readlines()
296

297
        utilities = defaultdict(lambda: [None, None, {}])
298
        lines = []
Stefan Behnel's avatar
Stefan Behnel committed
299
        tags = defaultdict(set)
300 301 302
        utility = type = None
        begin_lineno = 0

Mark Florisson's avatar
Mark Florisson committed
303
        for lineno, line in enumerate(all_lines):
304
            m = match_special(line)
Mark Florisson's avatar
Mark Florisson committed
305
            if m:
306
                if m.group('name'):
307 308 309 310 311 312
                    cls._add_utility(utility, type, lines, begin_lineno, tags)

                    begin_lineno = lineno + 1
                    del lines[:]
                    tags.clear()

313 314 315 316
                    name = m.group('name')
                    mtype = match_type(name)
                    if mtype:
                        name, type = mtype.groups()
317
                    else:
318
                        type = 'impl'
Stefan Behnel's avatar
Stefan Behnel committed
319
                    utility = utilities[name]
Mark Florisson's avatar
Mark Florisson committed
320
                else:
Stefan Behnel's avatar
Stefan Behnel committed
321 322
                    tags[m.group('tag')].add(m.group('value'))
                    lines.append('')  # keep line number correct
Mark Florisson's avatar
Mark Florisson committed
323
            else:
Stefan Behnel's avatar
Stefan Behnel committed
324
                lines.append(rstrip(strip_comments(line)))
Mark Florisson's avatar
Mark Florisson committed
325

326
        if utility is None:
327 328 329
            raise ValueError("Empty utility code file")

        # Don't forget to add the last utility code
330
        cls._add_utility(utility, type, lines, begin_lineno, tags)
331

Stefan Behnel's avatar
Stefan Behnel committed
332
        utilities = dict(utilities)  # un-defaultdict-ify
333 334 335
        cls._utility_cache[path] = utilities
        return utilities

336
    @classmethod
337
    def load(cls, util_code_name, from_file=None, **kwargs):
338
        """
339 340 341 342
        Load utility code from a file specified by from_file (relative to
        Cython/Utility) and name util_code_name.  If from_file is not given,
        load it from the file util_code_name.*.  There should be only one
        file matched by this pattern.
343
        """
344 345 346
        if '::' in util_code_name:
            from_file, util_code_name = util_code_name.rsplit('::', 1)
        if not from_file:
347 348
            utility_dir = get_utility_dir()
            prefix = util_code_name + '.'
349 350 351 352 353 354 355 356 357
            try:
                listing = os.listdir(utility_dir)
            except OSError:
                # XXX the code below assumes as 'zipimport.zipimporter' instance
                # XXX should be easy to generalize, but too lazy right now to write it
                import zipfile
                global __loader__
                loader = __loader__
                archive = loader.archive
Stefan Behnel's avatar
Stefan Behnel committed
358
                with closing(zipfile.ZipFile(archive)) as fileobj:
359 360 361 362 363
                    listing = [os.path.basename(name)
                               for name in fileobj.namelist()
                               if os.path.join(archive, name).startswith(utility_dir)]
            files = [filename for filename in listing
                     if filename.startswith(prefix)]
364 365 366 367 368
            if not files:
                raise ValueError("No match found for utility code " + util_code_name)
            if len(files) > 1:
                raise ValueError("More than one filename match found for utility code " + util_code_name)
            from_file = files[0]
369 370

        utilities = cls.load_utilities_from_file(from_file)
371
        proto, impl, tags = utilities[util_code_name]
372 373 374

        if tags:
            orig_kwargs = kwargs.copy()
Stefan Behnel's avatar
Stefan Behnel committed
375
            for name, values in tags.items():
376 377 378 379
                if name in kwargs:
                    continue
                # only pass lists when we have to: most argument expect one value or None
                if name == 'requires':
380 381 382 383 384 385 386
                    if orig_kwargs:
                        values = [cls.load(dep, from_file, **orig_kwargs)
                                  for dep in sorted(values)]
                    else:
                        # dependencies are rarely unique, so use load_cached() when we can
                        values = [cls.load_cached(dep, from_file)
                                  for dep in sorted(values)]
387 388 389
                elif not values:
                    values = None
                elif len(values) == 1:
390
                    values = list(values)[0]
391
                kwargs[name] = values
Mark Florisson's avatar
Mark Florisson committed
392

393 394 395 396
        if proto is not None:
            kwargs['proto'] = proto
        if impl is not None:
            kwargs['impl'] = impl
Mark Florisson's avatar
Mark Florisson committed
397

398 399
        if 'name' not in kwargs:
            kwargs['name'] = util_code_name
Mark Florisson's avatar
Mark Florisson committed
400

401 402 403 404 405
        if 'file' not in kwargs and from_file:
            kwargs['file'] = from_file
        return cls(**kwargs)

    @classmethod
406
    def load_cached(cls, utility_code_name, from_file=None, __cache={}):
407
        """
408
        Calls .load(), but using a per-type cache based on utility name and file name.
409
        """
410 411
        key = (cls, from_file, utility_code_name)
        try:
412
            return __cache[key]
413 414
        except KeyError:
            pass
415
        code = __cache[key] = cls.load(utility_code_name, from_file)
416
        return code
417

418 419 420 421 422 423 424
    @classmethod
    def load_as_string(cls, util_code_name, from_file=None, **kwargs):
        """
        Load a utility code as a string. Returns (proto, implementation)
        """
        util = cls.load(util_code_name, from_file, **kwargs)
        proto, impl = util.proto, util.impl
425
        return util.format_code(proto), util.format_code(impl)
426

427
    def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub):
428 429 430 431 432 433 434
        """
        Format a code section for output.
        """
        if code_string:
            code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n'
        return code_string

435
    def __str__(self):
436
        return "<%s(%s)>" % (type(self).__name__, self.name)
437

438
    def get_tree(self, **kwargs):
439
        pass
440

Stefan Behnel's avatar
Stefan Behnel committed
441

442
class UtilityCode(UtilityCodeBase):
443 444 445 446 447 448 449 450
    """
    Stores utility code to add during code generation.

    See GlobalState.put_utility_code.

    hashes/equals by instance

    proto           C prototypes
luz.paz's avatar
luz.paz committed
451
    impl            implementation code
452 453 454 455 456 457 458 459
    init            code to call on module initialization
    requires        utility code dependencies
    proto_block     the place in the resulting file where the prototype should
                    end up
    name            name of the utility code (or None)
    file            filename of the utility code file this utility was loaded
                    from (or None)
    """
460

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
461
    def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
462
                 proto_block='utility_code_proto', name=None, file=None):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
463
        # proto_block: Which code block to dump prototype in. See GlobalState.
464 465 466 467 468 469 470
        self.proto = proto
        self.impl = impl
        self.init = init
        self.cleanup = cleanup
        self.requires = requires
        self._cache = {}
        self.specialize_list = []
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
471
        self.proto_block = proto_block
472
        self.name = name
473
        self.file = file
474

475 476 477 478 479 480
    def __hash__(self):
        return hash((self.proto, self.impl))

    def __eq__(self, other):
        if self is other:
            return True
481 482
        self_type, other_type = type(self), type(other)
        if self_type is not other_type and not (isinstance(other, self_type) or isinstance(self, other_type)):
483 484 485 486 487 488 489 490 491 492 493 494 495 496 497
            return False

        self_proto = getattr(self, 'proto', None)
        other_proto = getattr(other, 'proto', None)
        return (self_proto, self.impl) == (other_proto, other.impl)

    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return s % context

    def specialize(self, pyrex_type=None, **data):
498 499
        # Dicts aren't hashable...
        if pyrex_type is not None:
500
            data['type'] = pyrex_type.empty_declaration_code()
Craig Citro's avatar
Craig Citro committed
501
            data['type_name'] = pyrex_type.specialization_name()
Stefan Behnel's avatar
Stefan Behnel committed
502
        key = tuple(sorted(data.items()))
503 504 505 506 507 508 509
        try:
            return self._cache[key]
        except KeyError:
            if self.requires is None:
                requires = None
            else:
                requires = [r.specialize(data) for r in self.requires]
510

511
            s = self._cache[key] = UtilityCode(
Stefan Behnel's avatar
Stefan Behnel committed
512 513 514 515 516 517
                self.none_or_sub(self.proto, data),
                self.none_or_sub(self.impl, data),
                self.none_or_sub(self.init, data),
                self.none_or_sub(self.cleanup, data),
                requires,
                self.proto_block)
518

519 520 521
            self.specialize_list.append(s)
            return s

522 523 524
    def inject_string_constants(self, impl, output):
        """Replace 'PYIDENT("xyz")' by a constant Python identifier cname.
        """
525
        if 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl:
526 527
            return False, impl

528 529
        replacements = {}
        def externalise(matchobj):
530
            key = matchobj.groups()
531
            try:
532
                cname = replacements[key]
533
            except KeyError:
534 535 536
                str_type, name = key
                cname = replacements[key] = output.get_py_string_const(
                        StringEncoding.EncodedString(name), identifier=str_type == 'IDENT').cname
537 538
            return cname

539 540
        impl = re.sub(r'PY(IDENT|UNICODE)\("([^"]+)"\)', externalise, impl)
        assert 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl
541
        return bool(replacements), impl
542

543 544 545 546 547 548
    def inject_unbound_methods(self, impl, output):
        """Replace 'UNBOUND_METHOD(type, "name")' by a constant Python identifier cname.
        """
        if 'CALL_UNBOUND_METHOD(' not in impl:
            return False, impl

549
        utility_code = set()
550
        def externalise(matchobj):
551 552 553 554 555 556 557 558 559 560 561 562
            type_cname, method_name, obj_cname, args = matchobj.groups()
            args = [arg.strip() for arg in args[1:].split(',')] if args else []
            assert len(args) < 3, "CALL_UNBOUND_METHOD() does not support %d call arguments" % len(args)
            return output.cached_unbound_method_call_code(obj_cname, type_cname, method_name, args)

        impl = re.sub(
            r'CALL_UNBOUND_METHOD\('
            r'([a-zA-Z_]+),'      # type cname
            r'\s*"([^"]+)",'      # method name
            r'\s*([^),]+)'        # object cname
            r'((?:,\s*[^),]+)*)'  # args*
            r'\)', externalise, impl)
563
        assert 'CALL_UNBOUND_METHOD(' not in impl
564 565 566 567

        for helper in sorted(utility_code):
            output.use_utility_code(UtilityCode.load_cached(helper, "ObjectHandling.c"))
        return bool(utility_code), impl
568

569 570 571 572 573 574 575 576 577 578 579 580
    def wrap_c_strings(self, impl):
        """Replace CSTRING('''xyz''') by a C compatible string
        """
        if 'CSTRING(' not in impl:
            return impl

        def split_string(matchobj):
            content = matchobj.group(1).replace('"', '\042')
            return ''.join(
                '"%s\\n"\n' % line if not line.endswith('\\') or line.endswith('\\\\') else '"%s"\n' % line[:-1]
                for line in content.splitlines())

581
        impl = re.sub(r'CSTRING\(\s*"""([^"]*(?:"[^"]+)*)"""\s*\)', split_string, impl)
582 583 584
        assert 'CSTRING(' not in impl
        return impl

585 586 587 588 589
    def put_code(self, output):
        if self.requires:
            for dependency in self.requires:
                output.use_utility_code(dependency)
        if self.proto:
590 591 592 593
            writer = output[self.proto_block]
            writer.putln("/* %s.proto */" % self.name)
            writer.put_or_include(
                self.format_code(self.proto), '%s_proto' % self.name)
594
        if self.impl:
595
            impl = self.format_code(self.wrap_c_strings(self.impl))
596 597
            is_specialised1, impl = self.inject_string_constants(impl, output)
            is_specialised2, impl = self.inject_unbound_methods(impl, output)
598
            writer = output['utility_code_def']
Stefan Behnel's avatar
Stefan Behnel committed
599
            writer.putln("/* %s */" % self.name)
600
            if not (is_specialised1 or is_specialised2):
601
                # no module specific adaptations => can be reused
602
                writer.put_or_include(impl, '%s_impl' % self.name)
603
            else:
604
                writer.put(impl)
605 606
        if self.init:
            writer = output['init_globals']
607
            writer.putln("/* %s.init */" % self.name)
608
            if isinstance(self.init, basestring):
609
                writer.put(self.format_code(self.init))
610 611
            else:
                self.init(writer, output.module_pos)
612 613
            writer.putln(writer.error_goto_if_PyErr(output.module_pos))
            writer.putln()
614 615
        if self.cleanup and Options.generate_cleanup_code:
            writer = output['cleanup_globals']
616
            writer.putln("/* %s.cleanup */" % self.name)
617
            if isinstance(self.cleanup, basestring):
618 619 620
                writer.put_or_include(
                    self.format_code(self.cleanup),
                    '%s_cleanup' % self.name)
621 622
            else:
                self.cleanup(writer, output.module_pos)
623 624


625
def sub_tempita(s, context, file=None, name=None):
626 627 628
    "Run tempita on string s with given context."
    if not s:
        return None
629

630 631 632 633
    if file:
        context['__name'] = "%s:%s" % (file, name)
    elif name:
        context['__name'] = name
634

635
    from ..Tempita import sub
636
    return sub(s, **context)
637

638

639
class TempitaUtilityCode(UtilityCode):
640
    def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs):
641 642
        if context is None:
            context = {}
643 644
        proto = sub_tempita(proto, context, file, name)
        impl = sub_tempita(impl, context, file, name)
645
        init = sub_tempita(init, context, file, name)
646
        super(TempitaUtilityCode, self).__init__(
647
            proto, impl, init=init, name=name, file=file, **kwargs)
648

649 650 651 652 653 654 655 656 657 658 659 660
    @classmethod
    def load_cached(cls, utility_code_name, from_file=None, context=None, __cache={}):
        context_key = tuple(sorted(context.items())) if context else None
        assert hash(context_key) is not None  # raise TypeError if not hashable
        key = (cls, from_file, utility_code_name, context_key)
        try:
            return __cache[key]
        except KeyError:
            pass
        code = __cache[key] = cls.load(utility_code_name, from_file, context=context)
        return code

661 662 663 664 665 666 667
    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return sub_tempita(s, context, self.file, self.name)
668 669 670 671 672 673 674


class LazyUtilityCode(UtilityCodeBase):
    """
    Utility code that calls a callback with the root code writer when
    available. Useful when you only have 'env' but not 'code'.
    """
675
    __name__ = '<lazy>'
676
    requires = None
677 678 679 680 681 682 683 684

    def __init__(self, callback):
        self.callback = callback

    def put_code(self, globalstate):
        utility = self.callback(globalstate.rootwriter)
        globalstate.use_utility_code(utility)

685

686 687 688 689 690 691 692 693 694
class FunctionState(object):
    # return_label     string          function return point label
    # error_label      string          error catch point label
    # continue_label   string          loop continue point label
    # break_label      string          loop break point label
    # return_from_error_cleanup_label string
    # label_counter    integer         counter for naming labels
    # in_try_finally   boolean         inside try of try...finally
    # exc_vars         (string * 3)    exception variables for reraise, or None
695
    # can_trace        boolean         line tracing is supported in the current context
696
    # scope            Scope           the scope object of the current function
697

698
    # Not used for now, perhaps later
699
    def __init__(self, owner, names_taken=set(), scope=None):
700
        self.names_taken = names_taken
701
        self.owner = owner
702
        self.scope = scope
703

704 705
        self.error_label = None
        self.label_counter = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
706
        self.labels_used = set()
707 708 709 710
        self.return_label = self.new_label()
        self.new_error_label()
        self.continue_label = None
        self.break_label = None
711
        self.yield_labels = []
712

713 714
        self.in_try_finally = 0
        self.exc_vars = None
715
        self.current_except = None
716
        self.can_trace = False
717
        self.gil_owned = True
718

719
        self.temps_allocated = [] # of (name, type, manage_ref, static)
720 721
        self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
        self.temps_used_type = {} # name -> (type, manage_ref)
722
        self.temp_counter = 0
723
        self.closure_temps = None
724

725 726 727 728
        # This is used to collect temporaries, useful to find out which temps
        # need to be privatized in parallel sections
        self.collect_temps_stack = []

729 730 731 732 733
        # This is used for the error indicator, which needs to be local to the
        # function. It used to be global, which relies on the GIL being held.
        # However, exceptions may need to be propagated through 'nogil'
        # sections, in which case we introduce a race condition.
        self.should_declare_error_indicator = False
734
        self.uses_error_indicator = False
735

736 737
    # labels

738
    def new_label(self, name=None):
739 740
        n = self.label_counter
        self.label_counter = n + 1
741 742 743 744
        label = "%s%d" % (Naming.label_prefix, n)
        if name is not None:
            label += '_' + name
        return label
745

746 747
    def new_yield_label(self, expr_type='yield'):
        label = self.new_label('resume_from_%s' % expr_type)
748 749 750 751
        num_and_label = (len(self.yield_labels) + 1, label)
        self.yield_labels.append(num_and_label)
        return num_and_label

752 753
    def new_error_label(self):
        old_err_lbl = self.error_label
754
        self.error_label = self.new_label('error')
755
        return old_err_lbl
756

757 758 759 760
    def get_loop_labels(self):
        return (
            self.continue_label,
            self.break_label)
761

762 763 764
    def set_loop_labels(self, labels):
        (self.continue_label,
         self.break_label) = labels
765

766 767 768
    def new_loop_labels(self):
        old_labels = self.get_loop_labels()
        self.set_loop_labels(
769
            (self.new_label("continue"),
Robert Bradshaw's avatar
Robert Bradshaw committed
770
             self.new_label("break")))
771
        return old_labels
772

773 774 775 776 777 778 779 780 781 782 783 784 785 786 787 788
    def get_all_labels(self):
        return (
            self.continue_label,
            self.break_label,
            self.return_label,
            self.error_label)

    def set_all_labels(self, labels):
        (self.continue_label,
         self.break_label,
         self.return_label,
         self.error_label) = labels

    def all_new_labels(self):
        old_labels = self.get_all_labels()
        new_labels = []
789
        for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']):
790
            if old_label:
791
                new_labels.append(self.new_label(name))
792 793 794 795
            else:
                new_labels.append(old_label)
        self.set_all_labels(new_labels)
        return old_labels
796

797
    def use_label(self, lbl):
Stefan Behnel's avatar
Stefan Behnel committed
798
        self.labels_used.add(lbl)
799

800 801 802
    def label_used(self, lbl):
        return lbl in self.labels_used

803 804
    # temp handling

805
    def allocate_temp(self, type, manage_ref, static=False):
806 807 808 809 810
        """
        Allocates a temporary (which may create a new one or get a previously
        allocated and released one of the same type). Type is simply registered
        and handed back, but will usually be a PyrexType.

811 812 813 814 815
        If type.is_pyobject, manage_ref comes into play. If manage_ref is set to
        True, the temp will be decref-ed on return statements and in exception
        handling clauses. Otherwise the caller has to deal with any reference
        counting of the variable.

816 817 818 819
        If not type.is_pyobject, then manage_ref will be ignored, but it
        still has to be passed. It is recommended to pass False by convention
        if it is known that type will never be a Python object.

820 821 822 823
        static=True marks the temporary declaration with "static".
        This is only used when allocating backing store for a module-level
        C array literals.

824 825
        A C string referring to the variable is returned.
        """
826
        if type.is_const and not type.is_reference:
827
            type = type.const_base_type
828
        elif type.is_reference and not type.is_fake_reference:
829
            type = type.ref_base_type
830
        if not type.is_pyobject and not type.is_memoryviewslice:
831 832 833
            # Make manage_ref canonical, so that manage_ref will always mean
            # a decref is needed.
            manage_ref = False
834

835
        freelist = self.temps_free.get((type, manage_ref))
836 837 838
        if freelist is not None and freelist[0]:
            result = freelist[0].pop()
            freelist[1].remove(result)
839
        else:
840 841 842
            while True:
                self.temp_counter += 1
                result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)
843
                if result not in self.names_taken: break
844
            self.temps_allocated.append((result, type, manage_ref, static))
845
        self.temps_used_type[result] = (type, manage_ref)
846
        if DebugFlags.debug_temp_code_comments:
847
            self.owner.putln("/* %s allocated (%s) */" % (result, type))
848 849 850 851

        if self.collect_temps_stack:
            self.collect_temps_stack[-1].add((result, type))

852 853 854 855 856 857 858
        return result

    def release_temp(self, name):
        """
        Releases a temporary so that it can be reused by other code needing
        a temp of the same type.
        """
859 860
        type, manage_ref = self.temps_used_type[name]
        freelist = self.temps_free.get((type, manage_ref))
861
        if freelist is None:
862
            freelist = ([], set())  # keep order in list and make lookups in set fast
863
            self.temps_free[(type, manage_ref)] = freelist
864
        if name in freelist[1]:
865
            raise RuntimeError("Temp %s freed twice!" % name)
866 867
        freelist[0].append(name)
        freelist[1].add(name)
868 869
        if DebugFlags.debug_temp_code_comments:
            self.owner.putln("/* %s released */" % name)
870

871
    def temps_in_use(self):
872
        """Return a list of (cname,type,manage_ref) tuples of temp names and their type
873 874 875
        that are currently in use.
        """
        used = []
876
        for name, type, manage_ref, static in self.temps_allocated:
877
            freelist = self.temps_free.get((type, manage_ref))
878
            if freelist is None or name not in freelist[1]:
879
                used.append((name, type, manage_ref and type.is_pyobject))
880 881
        return used

882
    def temps_holding_reference(self):
883
        """Return a list of (cname,type) tuples of temp names and their type
884 885
        that are currently in use. This includes only temps of a
        Python object type which owns its reference.
886 887
        """
        return [(name, type)
888
                for name, type, manage_ref in self.temps_in_use()
889
                if manage_ref  and type.is_pyobject]
890 891 892 893 894

    def all_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python objects.
        """
        return [(cname, type)
Stefan Behnel's avatar
Stefan Behnel committed
895 896
                for cname, type, manage_ref, static in self.temps_allocated
                if manage_ref]
897

898 899 900 901 902 903 904
    def all_free_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python
        objects that are not currently in use.  This is used by
        try-except and try-finally blocks to clean up temps in the
        error case.
        """
        return [(cname, type)
905
                for (type, manage_ref), freelist in self.temps_free.items() if manage_ref
906
                for cname in freelist[0]]
907

908 909 910 911
    def start_collecting_temps(self):
        """
        Useful to find out which temps were used in a code block
        """
Robert Bradshaw's avatar
Robert Bradshaw committed
912
        self.collect_temps_stack.append(set())
913 914 915

    def stop_collecting_temps(self):
        return self.collect_temps_stack.pop()
916

917 918 919
    def init_closure_temps(self, scope):
        self.closure_temps = ClosureTempAllocator(scope)

920

Stefan Behnel's avatar
Stefan Behnel committed
921 922 923 924 925 926 927
class NumConst(object):
    """Global info about a Python number constant held by GlobalState.

    cname       string
    value       string
    py_type     string     int, long, float
    value_code  string     evaluation code if different from value
928 929
    """

Stefan Behnel's avatar
Stefan Behnel committed
930
    def __init__(self, cname, value, py_type, value_code=None):
931 932
        self.cname = cname
        self.value = value
Stefan Behnel's avatar
Stefan Behnel committed
933 934 935
        self.py_type = py_type
        self.value_code = value_code or value

936

937 938 939 940 941 942 943 944 945 946
class PyObjectConst(object):
    """Global info about a generic constant held by GlobalState.
    """
    # cname       string
    # type        PyrexType

    def __init__(self, cname, type):
        self.cname = cname
        self.type = type

Stefan Behnel's avatar
Stefan Behnel committed
947

948
cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object,
949
               replace_identifier=object, find_alphanums=object)
950
possible_unicode_identifier = re.compile(br"(?![0-9])\w+$".decode('ascii'), re.U).match
Stefan Behnel's avatar
Stefan Behnel committed
951
possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match
952
replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub
953
find_alphanums = re.compile('([a-zA-Z0-9]+)').findall
954 955 956 957 958 959 960 961 962 963 964 965 966

class StringConst(object):
    """Global info about a C string constant held by GlobalState.
    """
    # cname            string
    # text             EncodedString or BytesLiteral
    # py_strings       {(identifier, encoding) : PyStringConst}

    def __init__(self, cname, text, byte_string):
        self.cname = cname
        self.text = text
        self.escaped_value = StringEncoding.escape_byte_string(byte_string)
        self.py_strings = None
967 968 969
        self.py_versions = []

    def add_py_version(self, version):
970
        if not version:
Stefan Behnel's avatar
Stefan Behnel committed
971
            self.py_versions = [2, 3]
972
        elif version not in self.py_versions:
973
            self.py_versions.append(version)
974

975 976
    def get_py_string_const(self, encoding, identifier=None,
                            is_str=False, py3str_cstring=None):
977 978 979
        py_strings = self.py_strings
        text = self.text

Stefan Behnel's avatar
Stefan Behnel committed
980
        is_str = bool(identifier or is_str)
981
        is_unicode = encoding is None and not is_str
982

983 984 985 986 987 988 989 990 991 992 993 994
        if encoding is None:
            # unicode string
            encoding_key = None
        else:
            # bytes or str
            encoding = encoding.lower()
            if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):
                encoding = None
                encoding_key = None
            else:
                encoding_key = ''.join(find_alphanums(encoding))

995 996 997 998 999 1000
        key = (is_str, is_unicode, encoding_key, py3str_cstring)
        if py_strings is not None:
            try:
                return py_strings[key]
            except KeyError:
                pass
1001
        else:
1002
            self.py_strings = {}
1003

1004 1005 1006
        if identifier:
            intern = True
        elif identifier is None:
1007
            if isinstance(text, bytes):
1008
                intern = bool(possible_bytes_identifier(text))
1009 1010
            else:
                intern = bool(possible_unicode_identifier(text))
1011 1012 1013
        else:
            intern = False
        if intern:
1014
            prefix = Naming.interned_prefixes['str']
1015 1016
        else:
            prefix = Naming.py_const_prefix
1017 1018 1019 1020 1021 1022 1023

        if encoding_key:
            encoding_prefix = '_%s' % encoding_key
        else:
            encoding_prefix = ''

        pystring_cname = "%s%s%s_%s" % (
1024 1025
            prefix,
            (is_str and 's') or (is_unicode and 'u') or 'b',
1026
            encoding_prefix,
1027 1028 1029 1030 1031
            self.cname[len(Naming.const_prefix):])

        py_string = PyStringConst(
            pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern)
        self.py_strings[key] = py_string
1032 1033 1034 1035 1036 1037
        return py_string

class PyStringConst(object):
    """Global info about a Python string constant held by GlobalState.
    """
    # cname       string
1038
    # py3str_cstring string
1039
    # encoding    string
1040
    # intern      boolean
1041 1042
    # is_unicode  boolean
    # is_str      boolean
1043

1044 1045
    def __init__(self, cname, encoding, is_unicode, is_str=False,
                 py3str_cstring=None, intern=False):
1046
        self.cname = cname
1047
        self.py3str_cstring = py3str_cstring
1048 1049 1050
        self.encoding = encoding
        self.is_str = is_str
        self.is_unicode = is_unicode
1051 1052
        self.intern = intern

Stefan Behnel's avatar
Stefan Behnel committed
1053 1054 1055
    def __lt__(self, other):
        return self.cname < other.cname

1056

1057 1058 1059 1060 1061 1062
class GlobalState(object):
    # filename_table   {string : int}  for finding filename table indexes
    # filename_list    [string]        filenames in filename table order
    # input_file_contents dict         contents (=list of lines) of any file that was used as input
    #                                  to create this output C code.  This is
    #                                  used to annotate the comments.
1063
    #
1064
    # utility_codes   set                IDs of used utility code (to avoid reinsertion)
1065 1066 1067 1068 1069 1070
    #
    # declared_cnames  {string:Entry}  used in a transition phase to merge pxd-declared
    #                                  constants etc. into the pyx-declared ones (i.e,
    #                                  check if constants are already added).
    #                                  In time, hopefully the literals etc. will be
    #                                  supplied directly instead.
1071
    #
1072
    # const_cnames_used  dict          global counter for unique constant identifiers
1073
    #
1074

1075 1076
    # parts            {string:CCodeWriter}

1077

1078 1079 1080 1081
    # interned_strings
    # consts
    # interned_nums

1082 1083 1084 1085 1086 1087
    # directives       set             Temporary variable used to track
    #                                  the current set of directives in the code generation
    #                                  process.

    directives = {}

1088 1089
    code_layout = [
        'h_code',
Robert Bradshaw's avatar
Robert Bradshaw committed
1090
        'filename_table',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1091
        'utility_code_proto_before_types',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1092 1093 1094 1095
        'numeric_typedefs',          # Let these detailed individual parts stay!,
        'complex_type_declarations', # as the proper solution is to make a full DAG...
        'type_declarations',         # More coarse-grained blocks would simply hide
        'utility_code_proto',        # the ugliness, not fix it
1096 1097
        'module_declarations',
        'typeinfo',
1098 1099
        'before_global_var',
        'global_var',
1100
        'string_decls',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1101
        'decls',
Jeroen Demeyer's avatar
Jeroen Demeyer committed
1102
        'late_includes',
1103
        'all_the_rest',
1104 1105
        'pystring_table',
        'cached_builtins',
1106
        'cached_constants',
1107 1108 1109 1110 1111
        'init_globals',
        'init_module',
        'cleanup_globals',
        'cleanup_module',
        'main_method',
1112 1113
        'utility_code_def',
        'end'
1114
    ]
1115

1116

1117
    def __init__(self, writer, module_node, code_config, common_utility_include_dir=None):
1118 1119 1120
        self.filename_table = {}
        self.filename_list = []
        self.input_file_contents = {}
Robert Bradshaw's avatar
Robert Bradshaw committed
1121
        self.utility_codes = set()
1122
        self.declared_cnames = {}
1123
        self.in_utility_code_generation = False
1124
        self.code_config = code_config
1125
        self.common_utility_include_dir = common_utility_include_dir
1126
        self.parts = {}
1127 1128
        self.module_node = module_node # because some utility code generation needs it
                                       # (generating backwards-compatible Get/ReleaseBuffer
1129

1130
        self.const_cnames_used = {}
1131
        self.string_const_index = {}
Nikita Nemkin's avatar
Nikita Nemkin committed
1132
        self.pyunicode_ptr_const_index = {}
Stefan Behnel's avatar
Stefan Behnel committed
1133
        self.num_const_index = {}
1134
        self.py_constants = []
1135
        self.cached_cmethods = {}
1136

1137
        writer.set_global_state(self)
1138
        self.rootwriter = writer
1139

1140 1141 1142 1143
    def initialize_main_c_code(self):
        rootwriter = self.rootwriter
        for part in self.code_layout:
            self.parts[part] = rootwriter.insertion_point()
1144

1145 1146 1147 1148 1149
        if not Options.cache_builtins:
            del self.parts['cached_builtins']
        else:
            w = self.parts['cached_builtins']
            w.enter_cfunc_scope()
1150
            w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) {")
1151

1152 1153 1154
        w = self.parts['cached_constants']
        w.enter_cfunc_scope()
        w.putln("")
1155
        w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) {")
1156
        w.put_declare_refcount_context()
1157 1158
        w.put_setup_refcount_context("__Pyx_InitCachedConstants")

1159 1160 1161
        w = self.parts['init_globals']
        w.enter_cfunc_scope()
        w.putln("")
1162
        w.putln("static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) {")
1163

1164 1165 1166 1167 1168 1169
        if not Options.generate_cleanup_code:
            del self.parts['cleanup_globals']
        else:
            w = self.parts['cleanup_globals']
            w.enter_cfunc_scope()
            w.putln("")
1170
            w.putln("static CYTHON_SMALL_CODE void __Pyx_CleanupGlobals(void) {")
1171

1172 1173 1174 1175
        code = self.parts['utility_code_proto']
        code.putln("")
        code.putln("/* --- Runtime support code (head) --- */")

1176
        code = self.parts['utility_code_def']
1177
        if self.code_config.emit_linenums:
1178 1179
            code.write('\n#line 1 "cython_utility"\n')
        code.putln("")
1180
        code.putln("/* --- Runtime support code --- */")
1181

1182
    def finalize_main_c_code(self):
1183 1184 1185 1186 1187 1188
        self.close_global_decls()

        #
        # utility_code_def
        #
        code = self.parts['utility_code_def']
1189 1190
        util = TempitaUtilityCode.load_cached("TypeConversions", "TypeConversion.c")
        code.put(util.format_code(util.impl))
1191 1192
        code.putln("")

1193 1194 1195
    def __getitem__(self, key):
        return self.parts[key]

1196 1197 1198
    #
    # Global constants, interned objects, etc.
    #
1199 1200
    def close_global_decls(self):
        # This is called when it is known that no more global declarations will
1201
        # declared.
1202
        self.generate_const_declarations()
1203
        if Options.cache_builtins:
1204
            w = self.parts['cached_builtins']
1205
            w.putln("return 0;")
1206 1207 1208
            if w.label_used(w.error_label):
                w.put_label(w.error_label)
                w.putln("return -1;")
1209 1210 1211
            w.putln("}")
            w.exit_cfunc_scope()

1212 1213 1214 1215 1216 1217 1218 1219 1220 1221
        w = self.parts['cached_constants']
        w.put_finish_refcount_context()
        w.putln("return 0;")
        if w.label_used(w.error_label):
            w.put_label(w.error_label)
            w.put_finish_refcount_context()
            w.putln("return -1;")
        w.putln("}")
        w.exit_cfunc_scope()

1222
        w = self.parts['init_globals']
1223
        w.putln("return 0;")
1224 1225 1226
        if w.label_used(w.error_label):
            w.put_label(w.error_label)
            w.putln("return -1;")
1227 1228
        w.putln("}")
        w.exit_cfunc_scope()
1229

1230 1231 1232 1233 1234
        if Options.generate_cleanup_code:
            w = self.parts['cleanup_globals']
            w.putln("}")
            w.exit_cfunc_scope()

1235 1236 1237 1238
        if Options.generate_cleanup_code:
            w = self.parts['cleanup_module']
            w.putln("}")
            w.exit_cfunc_scope()
1239

1240
    def put_pyobject_decl(self, entry):
1241
        self['global_var'].putln("static PyObject *%s;" % entry.cname)
1242

1243 1244
    # constant handling at code generation time

1245 1246 1247
    def get_cached_constants_writer(self):
        return self.parts['cached_constants']

1248
    def get_int_const(self, str_value, longness=False):
Stefan Behnel's avatar
Stefan Behnel committed
1249
        py_type = longness and 'long' or 'int'
1250
        try:
Stefan Behnel's avatar
Stefan Behnel committed
1251
            c = self.num_const_index[(str_value, py_type)]
1252
        except KeyError:
Stefan Behnel's avatar
Stefan Behnel committed
1253 1254 1255 1256 1257 1258 1259 1260
            c = self.new_num_const(str_value, py_type)
        return c

    def get_float_const(self, str_value, value_code):
        try:
            c = self.num_const_index[(str_value, 'float')]
        except KeyError:
            c = self.new_num_const(str_value, 'float', value_code)
1261 1262
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1263
    def get_py_const(self, type, prefix='', cleanup_level=None):
1264
        # create a new Python object constant
Stefan Behnel's avatar
Stefan Behnel committed
1265
        const = self.new_py_const(type, prefix)
1266
        if cleanup_level is not None \
Stefan Behnel's avatar
Stefan Behnel committed
1267
                and cleanup_level <= Options.generate_cleanup_code:
1268
            cleanup_writer = self.parts['cleanup_globals']
1269
            cleanup_writer.putln('Py_CLEAR(%s);' % const.cname)
1270
        return const
1271

1272
    def get_string_const(self, text, py_version=None):
1273 1274
        # return a C string constant, creating a new one if necessary
        if text.is_unicode:
1275
            byte_string = text.utf8encode()
1276 1277 1278 1279 1280 1281
        else:
            byte_string = text.byteencode()
        try:
            c = self.string_const_index[byte_string]
        except KeyError:
            c = self.new_string_const(text, byte_string)
1282
        c.add_py_version(py_version)
1283 1284
        return c

1285
    def get_pyunicode_ptr_const(self, text):
1286 1287 1288
        # return a Py_UNICODE[] constant, creating a new one if necessary
        assert text.is_unicode
        try:
Nikita Nemkin's avatar
Nikita Nemkin committed
1289
            c = self.pyunicode_ptr_const_index[text]
1290
        except KeyError:
Nikita Nemkin's avatar
Nikita Nemkin committed
1291
            c = self.pyunicode_ptr_const_index[text] = self.new_const_cname()
1292 1293
        return c

1294 1295
    def get_py_string_const(self, text, identifier=None,
                            is_str=False, unicode_value=None):
1296
        # return a Python string constant, creating a new one if necessary
1297 1298
        py3str_cstring = None
        if is_str and unicode_value is not None \
1299
               and unicode_value.utf8encode() != text.byteencode():
1300 1301 1302 1303
            py3str_cstring = self.get_string_const(unicode_value, py_version=3)
            c_string = self.get_string_const(text, py_version=2)
        else:
            c_string = self.get_string_const(text)
1304 1305
        py_string = c_string.get_py_string_const(
            text.encoding, identifier, is_str, py3str_cstring)
1306 1307
        return py_string

1308 1309 1310
    def get_interned_identifier(self, text):
        return self.get_py_string_const(text, identifier=True)

1311
    def new_string_const(self, text, byte_string):
Stefan Behnel's avatar
Stefan Behnel committed
1312
        cname = self.new_string_const_cname(byte_string)
1313 1314 1315 1316
        c = StringConst(cname, text, byte_string)
        self.string_const_index[byte_string] = c
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1317 1318 1319 1320
    def new_num_const(self, value, py_type, value_code=None):
        cname = self.new_num_const_cname(value, py_type)
        c = NumConst(cname, value, py_type, value_code)
        self.num_const_index[(value, py_type)] = c
1321 1322
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1323 1324
    def new_py_const(self, type, prefix=''):
        cname = self.new_const_cname(prefix)
1325 1326 1327 1328
        c = PyObjectConst(cname, type)
        self.py_constants.append(c)
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1329
    def new_string_const_cname(self, bytes_value):
1330
        # Create a new globally-unique nice name for a C string constant.
Stefan Behnel's avatar
Stefan Behnel committed
1331
        value = bytes_value.decode('ASCII', 'ignore')
1332
        return self.new_const_cname(value=value)
1333

Stefan Behnel's avatar
Stefan Behnel committed
1334 1335
    def new_num_const_cname(self, value, py_type):
        if py_type == 'long':
1336
            value += 'L'
1337 1338
            py_type = 'int'
        prefix = Naming.interned_prefixes[py_type]
Stefan Behnel's avatar
Stefan Behnel committed
1339
        cname = "%s%s" % (prefix, value)
1340
        cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_')
1341 1342
        return cname

1343 1344
    def new_const_cname(self, prefix='', value=''):
        value = replace_identifier('_', value)[:32].strip('_')
1345 1346 1347
        used = self.const_cnames_used
        name_suffix = value
        while name_suffix in used:
1348
            counter = used[value] = used[value] + 1
1349
            name_suffix = '%s_%d' % (value, counter)
1350
        used[name_suffix] = 1
1351 1352 1353 1354 1355
        if prefix:
            prefix = Naming.interned_prefixes[prefix]
        else:
            prefix = Naming.const_prefix
        return "%s%s" % (prefix, name_suffix)
1356

1357 1358
    def get_cached_unbound_method(self, type_cname, method_name):
        key = (type_cname, method_name)
1359 1360 1361 1362 1363 1364 1365
        try:
            cname = self.cached_cmethods[key]
        except KeyError:
            cname = self.cached_cmethods[key] = self.new_const_cname(
                'umethod', '%s_%s' % (type_cname, method_name))
        return cname

1366 1367 1368 1369 1370 1371 1372 1373 1374 1375 1376 1377
    def cached_unbound_method_call_code(self, obj_cname, type_cname, method_name, arg_cnames):
        # admittedly, not the best place to put this method, but it is reused by UtilityCode and ExprNodes ...
        utility_code_name = "CallUnboundCMethod%d" % len(arg_cnames)
        self.use_utility_code(UtilityCode.load_cached(utility_code_name, "ObjectHandling.c"))
        cache_cname = self.get_cached_unbound_method(type_cname, method_name)
        args = [obj_cname] + arg_cnames
        return "__Pyx_%s(&%s, %s)" % (
            utility_code_name,
            cache_cname,
            ', '.join(args),
        )

1378
    def add_cached_builtin_decl(self, entry):
1379
        if entry.is_builtin and entry.is_const:
1380 1381
            if self.should_declare(entry.cname, entry):
                self.put_pyobject_decl(entry)
1382
                w = self.parts['cached_builtins']
1383 1384 1385 1386
                condition = None
                if entry.name in non_portable_builtins_map:
                    condition, replacement = non_portable_builtins_map[entry.name]
                    w.putln('#if %s' % condition)
1387
                    self.put_cached_builtin_init(
1388
                        entry.pos, StringEncoding.EncodedString(replacement),
1389
                        entry.cname)
1390 1391 1392 1393
                    w.putln('#else')
                self.put_cached_builtin_init(
                    entry.pos, StringEncoding.EncodedString(entry.name),
                    entry.cname)
1394
                if condition:
1395 1396 1397 1398 1399
                    w.putln('#endif')

    def put_cached_builtin_init(self, pos, name, cname):
        w = self.parts['cached_builtins']
        interned_cname = self.get_interned_identifier(name).cname
1400 1401 1402
        self.use_utility_code(
            UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c"))
        w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % (
1403 1404 1405 1406
            cname,
            interned_cname,
            cname,
            w.error_goto(pos)))
1407 1408

    def generate_const_declarations(self):
1409
        self.generate_cached_methods_decls()
1410
        self.generate_string_constants()
Stefan Behnel's avatar
Stefan Behnel committed
1411
        self.generate_num_constants()
1412 1413 1414
        self.generate_object_constant_decls()

    def generate_object_constant_decls(self):
Stefan Behnel's avatar
Stefan Behnel committed
1415 1416
        consts = [(len(c.cname), c.cname, c)
                  for c in self.py_constants]
1417
        consts.sort()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1418
        decls_writer = self.parts['decls']
1419
        for _, cname, c in consts:
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1420
            decls_writer.putln(
1421
                "static %s;" % c.type.declaration_code(cname))
1422

1423 1424 1425 1426
    def generate_cached_methods_decls(self):
        if not self.cached_cmethods:
            return

1427 1428
        decl = self.parts['decls']
        init = self.parts['init_globals']
1429
        cnames = []
1430
        for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()):
1431 1432
            cnames.append(cname)
            method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname
1433 1434 1435 1436 1437
            decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % (
                cname, method_name_cname))
            # split type reference storage as it might not be static
            init.putln('%s.type = (PyObject*)&%s;' % (
                cname, type_cname))
1438 1439 1440 1441 1442 1443

        if Options.generate_cleanup_code:
            cleanup = self.parts['cleanup_globals']
            for cname in cnames:
                cleanup.putln("Py_CLEAR(%s.method);" % cname)

1444
    def generate_string_constants(self):
1445
        c_consts = [(len(c.cname), c.cname, c) for c in self.string_const_index.values()]
1446 1447
        c_consts.sort()
        py_strings = []
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1448

1449
        decls_writer = self.parts['string_decls']
1450
        for _, cname, c in c_consts:
1451 1452 1453 1454 1455
            conditional = False
            if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions):
                conditional = True
                decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % (
                    (2 in c.py_versions) and '<' or '>='))
1456
            decls_writer.putln('static const char %s[] = "%s";' % (
1457
                cname, StringEncoding.split_string_literal(c.escaped_value)))
1458 1459
            if conditional:
                decls_writer.putln("#endif")
1460
            if c.py_strings is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1461
                for py_string in c.py_strings.values():
1462 1463
                    py_strings.append((c.cname, len(py_string.cname), py_string))

1464
        for c, cname in sorted(self.pyunicode_ptr_const_index.items()):
1465
            utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c)
1466 1467
            if utf16_array:
                # Narrow and wide representations differ
Nikita Nemkin's avatar
Nikita Nemkin committed
1468
                decls_writer.putln("#ifdef Py_UNICODE_WIDE")
1469 1470 1471 1472 1473
            decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf32_array))
            if utf16_array:
                decls_writer.putln("#else")
                decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array))
                decls_writer.putln("#endif")
1474

1475
        if py_strings:
1476
            self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c"))
1477
            py_strings.sort()
1478 1479
            w = self.parts['pystring_table']
            w.putln("")
Stefan Behnel's avatar
Stefan Behnel committed
1480
            w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname)
1481
            for c_cname, _, py_string in py_strings:
1482
                if not py_string.is_str or not py_string.encoding or \
Stefan Behnel's avatar
Stefan Behnel committed
1483 1484
                        py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',
                                               'UTF8', 'UTF-8'):
1485 1486 1487 1488
                    encoding = '0'
                else:
                    encoding = '"%s"' % py_string.encoding.lower()

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1489
                decls_writer.putln(
1490
                    "static PyObject *%s;" % py_string.cname)
1491 1492
                if py_string.py3str_cstring:
                    w.putln("#if PY_MAJOR_VERSION >= 3")
Stefan Behnel's avatar
Stefan Behnel committed
1493
                    w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1494 1495 1496
                        py_string.cname,
                        py_string.py3str_cstring.cname,
                        py_string.py3str_cstring.cname,
1497 1498
                        '0', 1, 0,
                        py_string.intern
1499 1500
                        ))
                    w.putln("#else")
Stefan Behnel's avatar
Stefan Behnel committed
1501
                w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1502 1503 1504
                    py_string.cname,
                    c_cname,
                    c_cname,
1505 1506 1507 1508
                    encoding,
                    py_string.is_unicode,
                    py_string.is_str,
                    py_string.intern
1509
                    ))
1510 1511
                if py_string.py3str_cstring:
                    w.putln("#endif")
1512
            w.putln("{0, 0, 0, 0, 0, 0, 0}")
1513
            w.putln("};")
1514

1515 1516
            init_globals = self.parts['init_globals']
            init_globals.putln(
1517 1518
                "if (__Pyx_InitStrings(%s) < 0) %s;" % (
                    Naming.stringtab_cname,
1519
                    init_globals.error_goto(self.module_pos)))
1520

Stefan Behnel's avatar
Stefan Behnel committed
1521
    def generate_num_constants(self):
1522
        consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c)
Stefan Behnel's avatar
Stefan Behnel committed
1523
                  for c in self.num_const_index.values()]
1524
        consts.sort()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1525
        decls_writer = self.parts['decls']
Stefan Behnel's avatar
Stefan Behnel committed
1526
        init_globals = self.parts['init_globals']
1527
        for py_type, _, _, value, value_code, c in consts:
1528
            cname = c.cname
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1529
            decls_writer.putln("static PyObject *%s;" % cname)
Stefan Behnel's avatar
Stefan Behnel committed
1530
            if py_type == 'float':
Stefan Behnel's avatar
Stefan Behnel committed
1531
                function = 'PyFloat_FromDouble(%s)'
Stefan Behnel's avatar
Stefan Behnel committed
1532
            elif py_type == 'long':
Stefan Behnel's avatar
Stefan Behnel committed
1533
                function = 'PyLong_FromString((char *)"%s", 0, 0)'
1534
            elif Utils.long_literal(value):
Stefan Behnel's avatar
Stefan Behnel committed
1535
                function = 'PyInt_FromString((char *)"%s", 0, 0)'
1536 1537
            elif len(value.lstrip('-')) > 4:
                function = "PyInt_FromLong(%sL)"
1538
            else:
Stefan Behnel's avatar
Stefan Behnel committed
1539 1540 1541
                function = "PyInt_FromLong(%s)"
            init_globals.putln('%s = %s; %s' % (
                cname, function % value_code,
1542
                init_globals.error_goto_if_null(cname, self.module_pos)))
1543

1544 1545 1546
    # The functions below are there in a transition phase only
    # and will be deprecated. They are called from Nodes.BlockNode.
    # The copy&paste duplication is intentional in order to be able
1547
    # to see quickly how BlockNode worked, until this is replaced.
1548

1549 1550 1551
    def should_declare(self, cname, entry):
        if cname in self.declared_cnames:
            other = self.declared_cnames[cname]
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1552
            assert str(entry.type) == str(other.type)
1553 1554 1555 1556 1557 1558 1559 1560 1561
            assert entry.init == other.init
            return False
        else:
            self.declared_cnames[cname] = entry
            return True

    #
    # File name state
    #
1562

1563
    def lookup_filename(self, source_desc):
1564
        entry = source_desc.get_filenametable_entry()
1565
        try:
1566
            index = self.filename_table[entry]
1567 1568
        except KeyError:
            index = len(self.filename_list)
1569
            self.filename_list.append(source_desc)
1570
            self.filename_table[entry] = index
1571 1572 1573 1574 1575 1576
        return index

    def commented_file_contents(self, source_desc):
        try:
            return self.input_file_contents[source_desc]
        except KeyError:
1577 1578 1579 1580
            pass
        source_file = source_desc.get_lines(encoding='ASCII',
                                            error_handling='ignore')
        try:
1581 1582
            F = [u' * ' + line.rstrip().replace(
                    u'*/', u'*[inserted by cython to avoid comment closer]/'
1583 1584
                    ).replace(
                    u'/*', u'/[inserted by cython to avoid comment start]*'
1585
                    )
1586 1587 1588 1589 1590 1591 1592
                 for line in source_file]
        finally:
            if hasattr(source_file, 'close'):
                source_file.close()
        if not F: F.append(u'')
        self.input_file_contents[source_desc] = F
        return F
1593

1594 1595 1596
    #
    # Utility code state
    #
1597

1598
    def use_utility_code(self, utility_code):
1599
        """
1600 1601 1602 1603
        Adds code to the C file. utility_code should
        a) implement __eq__/__hash__ for the purpose of knowing whether the same
           code has already been included
        b) implement put_code, which takes a globalstate instance
1604

1605
        See UtilityCode.
1606
        """
1607
        if utility_code and utility_code not in self.utility_codes:
1608 1609
            self.utility_codes.add(utility_code)
            utility_code.put_code(self)
1610

1611 1612 1613 1614 1615 1616 1617 1618
    def use_entry_utility_code(self, entry):
        if entry is None:
            return
        if entry.utility_code:
            self.use_utility_code(entry.utility_code)
        if entry.utility_code_definition:
            self.use_utility_code(entry.utility_code_definition)

1619

1620 1621
def funccontext_property(func):
    name = func.__name__
1622
    attribute_of = operator.attrgetter(name)
1623
    def get(self):
1624
        return attribute_of(self.funcstate)
1625
    def set(self, value):
1626
        setattr(self.funcstate, name, value)
1627 1628
    return property(get, set)

1629

1630 1631 1632 1633 1634 1635 1636 1637 1638 1639 1640
class CCodeConfig(object):
    # emit_linenums       boolean         write #line pragmas?
    # emit_code_comments  boolean         copy the original code into C comments?
    # c_line_in_traceback boolean         append the c file and line number to the traceback for exceptions?

    def __init__(self, emit_linenums=True, emit_code_comments=True, c_line_in_traceback=True):
        self.emit_code_comments = emit_code_comments
        self.emit_linenums = emit_linenums
        self.c_line_in_traceback = c_line_in_traceback


1641
class CCodeWriter(object):
1642
    """
1643
    Utility class to output C code.
1644

1645
    When creating an insertion point one must care about the state that is
1646
    kept:
1647 1648
    - formatting state (level, bol) is cloned and used in insertion points
      as well
1649 1650
    - labels, temps, exc_vars: One must construct a scope in which these can
      exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for
luz.paz's avatar
luz.paz committed
1651
      sanity checking and forward compatibility). Created insertion points
1652 1653 1654 1655
      looses this scope and cannot access it.
    - marker: Not copied to insertion point
    - filename_table, filename_list, input_file_contents: All codewriters
      coming from the same root share the same instances simultaneously.
1656
    """
1657

1658 1659 1660 1661 1662 1663 1664 1665 1666 1667 1668 1669
    # f                   file            output file
    # buffer              StringIOTree

    # level               int             indentation level
    # bol                 bool            beginning of line?
    # marker              string          comment to emit before next line
    # funcstate           FunctionState   contains state local to a C function used for code
    #                                     generation (labels and temps state etc.)
    # globalstate         GlobalState     contains state global for a C file (input file info,
    #                                     utility code, declared constants etc.)
    # pyclass_stack       list            used during recursive code generation to pass information
    #                                     about the current class one is in
1670
    # code_config         CCodeConfig     configuration options for the C code writer
1671

1672
    @cython.locals(create_from='CCodeWriter')
1673
    def __init__(self, create_from=None, buffer=None, copy_formatting=False):
1674 1675
        if buffer is None: buffer = StringIOTree()
        self.buffer = buffer
1676 1677
        self.last_pos = None
        self.last_marked_pos = None
1678
        self.pyclass_stack = []
1679

1680
        self.funcstate = None
1681 1682
        self.globalstate = None
        self.code_config = None
1683
        self.level = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
1684
        self.call_level = 0
1685
        self.bol = 1
1686 1687

        if create_from is not None:
1688
            # Use same global state
1689
            self.set_global_state(create_from.globalstate)
1690
            self.funcstate = create_from.funcstate
1691
            # Clone formatting state
1692 1693 1694
            if copy_formatting:
                self.level = create_from.level
                self.bol = create_from.bol
Robert Bradshaw's avatar
Robert Bradshaw committed
1695
                self.call_level = create_from.call_level
1696 1697
            self.last_pos = create_from.last_pos
            self.last_marked_pos = create_from.last_marked_pos
1698

1699
    def create_new(self, create_from, buffer, copy_formatting):
1700 1701
        # polymorphic constructor -- very slightly more versatile
        # than using __class__
1702
        result = CCodeWriter(create_from, buffer, copy_formatting)
1703
        return result
1704

1705 1706 1707 1708 1709
    def set_global_state(self, global_state):
        assert self.globalstate is None  # prevent overwriting once it's set
        self.globalstate = global_state
        self.code_config = global_state.code_config

1710 1711 1712
    def copyto(self, f):
        self.buffer.copyto(f)

1713 1714 1715 1716
    def getvalue(self):
        return self.buffer.getvalue()

    def write(self, s):
1717
        # also put invalid markers (lineno 0), to indicate that those lines
Mark Florisson's avatar
Mark Florisson committed
1718
        # have no Cython source code correspondence
1719
        cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0
Mark Florisson's avatar
Mark Florisson committed
1720
        self.buffer.markers.extend([cython_lineno] * s.count('\n'))
1721
        self.buffer.write(s)
1722 1723

    def insertion_point(self):
1724
        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
1725 1726
        return other

1727 1728 1729 1730 1731
    def new_writer(self):
        """
        Creates a new CCodeWriter connected to the same global state, which
        can later be inserted using insert.
        """
1732
        return CCodeWriter(create_from=self)
1733 1734 1735 1736 1737 1738 1739 1740 1741 1742

    def insert(self, writer):
        """
        Inserts the contents of another code writer (created with
        the same global state) in the current location.

        It is ok to write to the inserted writer also after insertion.
        """
        assert writer.globalstate is self.globalstate
        self.buffer.insert(writer.buffer)
1743 1744

    # Properties delegated to function scope
1745 1746 1747 1748 1749 1750 1751 1752 1753 1754 1755 1756 1757 1758 1759 1760
    @funccontext_property
    def label_counter(self): pass
    @funccontext_property
    def return_label(self): pass
    @funccontext_property
    def error_label(self): pass
    @funccontext_property
    def labels_used(self): pass
    @funccontext_property
    def continue_label(self): pass
    @funccontext_property
    def break_label(self): pass
    @funccontext_property
    def return_from_error_cleanup_label(self): pass
    @funccontext_property
    def yield_labels(self): pass
1761 1762

    # Functions delegated to function scope
Dag Sverre Seljebotn's avatar
merge  
Dag Sverre Seljebotn committed
1763
    def new_label(self, name=None):    return self.funcstate.new_label(name)
1764
    def new_error_label(self):         return self.funcstate.new_error_label()
1765
    def new_yield_label(self, *args):  return self.funcstate.new_yield_label(*args)
1766 1767 1768 1769 1770 1771 1772 1773
    def get_loop_labels(self):         return self.funcstate.get_loop_labels()
    def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)
    def new_loop_labels(self):         return self.funcstate.new_loop_labels()
    def get_all_labels(self):          return self.funcstate.get_all_labels()
    def set_all_labels(self, labels):  return self.funcstate.set_all_labels(labels)
    def all_new_labels(self):          return self.funcstate.all_new_labels()
    def use_label(self, lbl):          return self.funcstate.use_label(lbl)
    def label_used(self, lbl):         return self.funcstate.label_used(lbl)
1774 1775


1776 1777
    def enter_cfunc_scope(self, scope=None):
        self.funcstate = FunctionState(self, scope=scope)
1778

1779
    def exit_cfunc_scope(self):
1780
        self.funcstate = None
1781

1782 1783
    # constant handling

Stefan Behnel's avatar
Stefan Behnel committed
1784
    def get_py_int(self, str_value, longness):
1785 1786
        return self.globalstate.get_int_const(str_value, longness).cname

Stefan Behnel's avatar
Stefan Behnel committed
1787 1788 1789
    def get_py_float(self, str_value, value_code):
        return self.globalstate.get_float_const(str_value, value_code).cname

1790 1791
    def get_py_const(self, type, prefix='', cleanup_level=None):
        return self.globalstate.get_py_const(type, prefix, cleanup_level).cname
1792

1793 1794 1795
    def get_string_const(self, text):
        return self.globalstate.get_string_const(text).cname

1796 1797
    def get_pyunicode_ptr_const(self, text):
        return self.globalstate.get_pyunicode_ptr_const(text)
1798

1799 1800 1801 1802
    def get_py_string_const(self, text, identifier=None,
                            is_str=False, unicode_value=None):
        return self.globalstate.get_py_string_const(
            text, identifier, is_str, unicode_value).cname
1803

1804 1805 1806
    def get_argument_default_const(self, type):
        return self.globalstate.get_py_const(type).cname

1807 1808 1809 1810
    def intern(self, text):
        return self.get_py_string_const(text)

    def intern_identifier(self, text):
1811
        return self.get_py_string_const(text, identifier=True)
1812

1813 1814 1815
    def get_cached_constants_writer(self):
        return self.globalstate.get_cached_constants_writer()

1816 1817
    # code generation

Stefan Behnel's avatar
Stefan Behnel committed
1818
    def putln(self, code="", safe=False):
1819
        if self.last_pos and self.bol:
William Stein's avatar
William Stein committed
1820
            self.emit_marker()
1821
        if self.code_config.emit_linenums and self.last_marked_pos:
1822 1823
            source_desc, line, _ = self.last_marked_pos
            self.write('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description()))
William Stein's avatar
William Stein committed
1824
        if code:
1825 1826 1827 1828
            if safe:
                self.put_safe(code)
            else:
                self.put(code)
1829
        self.write("\n")
William Stein's avatar
William Stein committed
1830
        self.bol = 1
1831

1832
    def mark_pos(self, pos, trace=True):
1833 1834 1835 1836
        if pos is None:
            return
        if self.last_marked_pos and self.last_marked_pos[:2] == pos[:2]:
            return
1837
        self.last_pos = (pos, trace)
1838

William Stein's avatar
William Stein committed
1839
    def emit_marker(self):
1840 1841
        pos, trace = self.last_pos
        self.last_marked_pos = pos
1842
        self.last_pos = None
1843
        self.write("\n")
1844
        if self.code_config.emit_code_comments:
1845 1846
            self.indent()
            self.write("/* %s */\n" % self._build_marker(pos))
1847
        if trace and self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']:
1848
            self.indent()
1849 1850 1851 1852 1853 1854 1855 1856 1857 1858 1859
            self.write('__Pyx_TraceLine(%d,%d,%s)\n' % (
                pos[1], not self.funcstate.gil_owned, self.error_goto(pos)))

    def _build_marker(self, pos):
        source_desc, line, col = pos
        assert isinstance(source_desc, SourceDescriptor)
        contents = self.globalstate.commented_file_contents(source_desc)
        lines = contents[max(0, line-3):line]  # line numbers start at 1
        lines[-1] += u'             # <<<<<<<<<<<<<<'
        lines += contents[line:line+2]
        return u'"%s":%d\n%s\n' % (source_desc.get_escaped_description(), line, u'\n'.join(lines))
William Stein's avatar
William Stein committed
1860

1861 1862 1863 1864 1865
    def put_safe(self, code):
        # put code, but ignore {}
        self.write(code)
        self.bol = 0

1866
    def put_or_include(self, code, name):
Stefan Behnel's avatar
Stefan Behnel committed
1867
        include_dir = self.globalstate.common_utility_include_dir
1868
        if include_dir and len(code) > 1024:
Stefan Behnel's avatar
Stefan Behnel committed
1869
            include_file = "%s_%s.h" % (
1870
                name, hashlib.md5(code.encode('utf8')).hexdigest())
Stefan Behnel's avatar
Stefan Behnel committed
1871 1872 1873
            path = os.path.join(include_dir, include_file)
            if not os.path.exists(path):
                tmp_path = '%s.tmp%s' % (path, os.getpid())
Stefan Behnel's avatar
Stefan Behnel committed
1874
                with closing(Utils.open_new_file(tmp_path)) as f:
Stefan Behnel's avatar
Stefan Behnel committed
1875
                    f.write(code)
1876
                shutil.move(tmp_path, path)
Stefan Behnel's avatar
Stefan Behnel committed
1877 1878
            code = '#include "%s"\n' % path
        self.put(code)
1879

William Stein's avatar
William Stein committed
1880
    def put(self, code):
1881
        fix_indent = False
1882 1883 1884 1885 1886 1887 1888 1889 1890 1891 1892 1893
        if "{" in code:
            dl = code.count("{")
        else:
            dl = 0
        if "}" in code:
            dl -= code.count("}")
            if dl < 0:
                self.level += dl
            elif dl == 0 and code[0] == "}":
                # special cases like "} else {" need a temporary dedent
                fix_indent = True
                self.level -= 1
William Stein's avatar
William Stein committed
1894 1895
        if self.bol:
            self.indent()
1896
        self.write(code)
William Stein's avatar
William Stein committed
1897 1898 1899
        self.bol = 0
        if dl > 0:
            self.level += dl
1900
        elif fix_indent:
1901 1902
            self.level += 1

Mark Florisson's avatar
Mark Florisson committed
1903
    def putln_tempita(self, code, **context):
1904
        from ..Tempita import sub
1905
        self.putln(sub(code, **context))
Mark Florisson's avatar
Mark Florisson committed
1906 1907

    def put_tempita(self, code, **context):
1908
        from ..Tempita import sub
1909
        self.put(sub(code, **context))
Mark Florisson's avatar
Mark Florisson committed
1910

William Stein's avatar
William Stein committed
1911
    def increase_indent(self):
Stefan Behnel's avatar
Stefan Behnel committed
1912
        self.level += 1
1913

William Stein's avatar
William Stein committed
1914
    def decrease_indent(self):
Stefan Behnel's avatar
Stefan Behnel committed
1915
        self.level -= 1
1916

William Stein's avatar
William Stein committed
1917 1918 1919
    def begin_block(self):
        self.putln("{")
        self.increase_indent()
1920

William Stein's avatar
William Stein committed
1921 1922 1923
    def end_block(self):
        self.decrease_indent()
        self.putln("}")
1924

William Stein's avatar
William Stein committed
1925
    def indent(self):
1926
        self.write("  " * self.level)
1927

1928 1929 1930
    def get_py_version_hex(self, pyversion):
        return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]

William Stein's avatar
William Stein committed
1931
    def put_label(self, lbl):
1932
        if lbl in self.funcstate.labels_used:
1933
            self.putln("%s:;" % lbl)
1934

1935
    def put_goto(self, lbl):
1936
        self.funcstate.use_label(lbl)
1937
        self.putln("goto %s;" % lbl)
1938

1939
    def put_var_declaration(self, entry, storage_class="",
Stefan Behnel's avatar
Stefan Behnel committed
1940
                            dll_linkage=None, definition=True):
1941
        #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
1942 1943
        if entry.visibility == 'private' and not (definition or entry.defined_in_pxd):
            #print "...private and not definition, skipping", entry.cname ###
1944
            return
1945 1946
        if entry.visibility == "private" and not entry.used:
            #print "...private and not used, skipping", entry.cname ###
William Stein's avatar
William Stein committed
1947 1948 1949
            return
        if storage_class:
            self.put("%s " % storage_class)
1950 1951
        if not entry.cf_used:
            self.put('CYTHON_UNUSED ')
1952
        self.put(entry.type.declaration_code(
Stefan Behnel's avatar
Stefan Behnel committed
1953
            entry.cname, dll_linkage=dll_linkage))
William Stein's avatar
William Stein committed
1954
        if entry.init is not None:
1955
            self.put_safe(" = %s" % entry.type.literal_code(entry.init))
1956
        elif entry.type.is_pyobject:
Stefan Behnel's avatar
Stefan Behnel committed
1957
            self.put(" = NULL")
William Stein's avatar
William Stein committed
1958
        self.putln(";")
1959 1960

    def put_temp_declarations(self, func_context):
1961
        for name, type, manage_ref, static in func_context.temps_allocated:
1962 1963 1964
            decl = type.declaration_code(name)
            if type.is_pyobject:
                self.putln("%s = NULL;" % decl)
1965
            elif type.is_memoryviewslice:
1966
                from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
1967
                self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init))
1968
            else:
1969
                self.putln("%s%s;" % (static and "static " or "", decl))
1970

1971
        if func_context.should_declare_error_indicator:
1972 1973 1974 1975
            if self.funcstate.uses_error_indicator:
                unused = ''
            else:
                unused = 'CYTHON_UNUSED '
1976
            # Initialize these variables to silence compiler warnings
1977 1978 1979
            self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname))
            self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cname))
            self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname))
1980

1981 1982 1983 1984
    def put_generated_by(self):
        self.putln("/* Generated by Cython %s */" % Version.watermark)
        self.putln("")

1985 1986 1987
    def put_h_guard(self, guard):
        self.putln("#ifndef %s" % guard)
        self.putln("#define %s" % guard)
1988

1989 1990 1991 1992 1993 1994
    def unlikely(self, cond):
        if Options.gcc_branch_hints:
            return 'unlikely(%s)' % cond
        else:
            return cond

1995 1996 1997 1998 1999
    def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper):
        if not modifiers:
            return ''
        return '%s ' % ' '.join([mapper(m,m) for m in modifiers])

2000 2001
    # Python objects and reference counting

William Stein's avatar
William Stein committed
2002 2003
    def entry_as_pyobject(self, entry):
        type = entry.type
2004 2005
        if (not entry.is_self_arg and not entry.type.is_complete()
            or entry.type.is_extension_type):
William Stein's avatar
William Stein committed
2006 2007 2008
            return "(PyObject *)" + entry.cname
        else:
            return entry.cname
2009

William Stein's avatar
William Stein committed
2010
    def as_pyobject(self, cname, type):
2011
        from .PyrexTypes import py_object_type, typecast
William Stein's avatar
William Stein committed
2012
        return typecast(py_object_type, type, cname)
2013

2014
    def put_gotref(self, cname):
2015
        self.putln("__Pyx_GOTREF(%s);" % cname)
2016

2017 2018
    def put_giveref(self, cname):
        self.putln("__Pyx_GIVEREF(%s);" % cname)
2019

2020 2021 2022
    def put_xgiveref(self, cname):
        self.putln("__Pyx_XGIVEREF(%s);" % cname)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2023 2024 2025
    def put_xgotref(self, cname):
        self.putln("__Pyx_XGOTREF(%s);" % cname)

2026 2027 2028 2029 2030
    def put_incref(self, cname, type, nanny=True):
        if nanny:
            self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
        else:
            self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))
2031

2032
    def put_decref(self, cname, type, nanny=True):
2033
        self._put_decref(cname, type, nanny, null_check=False, clear=False)
2034 2035 2036 2037

    def put_var_gotref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry))
2038

2039 2040 2041 2042
    def put_var_giveref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry))

2043 2044 2045 2046
    def put_var_xgotref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry))

2047 2048 2049 2050
    def put_var_xgiveref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry))

2051
    def put_var_incref(self, entry, nanny=True):
William Stein's avatar
William Stein committed
2052
        if entry.type.is_pyobject:
2053 2054 2055 2056
            if nanny:
                self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))
            else:
                self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry))
2057

2058 2059 2060 2061
    def put_var_xincref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry))

2062 2063 2064
    def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False):
        self._put_decref(cname, type, nanny, null_check=False,
                         clear=True, clear_before_decref=clear_before_decref)
2065

2066
    def put_xdecref(self, cname, type, nanny=True, have_gil=True):
2067 2068
        self._put_decref(cname, type, nanny, null_check=True,
                         have_gil=have_gil, clear=False)
2069

2070 2071 2072
    def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False):
        self._put_decref(cname, type, nanny, null_check=True,
                         clear=True, clear_before_decref=clear_before_decref)
2073

2074 2075
    def _put_decref(self, cname, type, nanny=True, null_check=False,
                    have_gil=True, clear=False, clear_before_decref=False):
2076
        if type.is_memoryviewslice:
2077
            self.put_xdecref_memoryviewslice(cname, have_gil=have_gil)
2078 2079
            return

2080 2081
        prefix = '__Pyx' if nanny else 'Py'
        X = 'X' if null_check else ''
2082 2083 2084 2085 2086 2087 2088 2089 2090

        if clear:
            if clear_before_decref:
                if not nanny:
                    X = ''  # CPython doesn't have a Py_XCLEAR()
                self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname))
            else:
                self.putln("%s_%sDECREF(%s); %s = 0;" % (
                    prefix, X, self.as_pyobject(cname, type), cname))
2091
        else:
2092 2093
            self.putln("%s_%sDECREF(%s);" % (
                prefix, X, self.as_pyobject(cname, type)))
William Stein's avatar
William Stein committed
2094

2095 2096 2097 2098 2099 2100
    def put_decref_set(self, cname, rhs_cname):
        self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname))

    def put_xdecref_set(self, cname, rhs_cname):
        self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname))

William Stein's avatar
William Stein committed
2101 2102
    def put_var_decref(self, entry):
        if entry.type.is_pyobject:
2103
            self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
2104

2105
    def put_var_xdecref(self, entry, nanny=True):
William Stein's avatar
William Stein committed
2106
        if entry.type.is_pyobject:
2107 2108 2109 2110
            if nanny:
                self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
            else:
                self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry))
2111

2112 2113 2114
    def put_var_decref_clear(self, entry):
        self._put_var_decref_clear(entry, null_check=False)

William Stein's avatar
William Stein committed
2115
    def put_var_xdecref_clear(self, entry):
2116 2117 2118
        self._put_var_decref_clear(entry, null_check=True)

    def _put_var_decref_clear(self, entry, null_check):
William Stein's avatar
William Stein committed
2119
        if entry.type.is_pyobject:
2120 2121 2122 2123 2124 2125 2126 2127 2128
            if entry.in_closure:
                # reset before DECREF to make sure closure state is
                # consistent during call to DECREF()
                self.putln("__Pyx_%sCLEAR(%s);" % (
                    null_check and 'X' or '',
                    entry.cname))
            else:
                self.putln("__Pyx_%sDECREF(%s); %s = 0;" % (
                    null_check and 'X' or '',
2129
                    self.entry_as_pyobject(entry),
2130
                    entry.cname))
2131

2132
    def put_var_decrefs(self, entries, used_only = 0):
William Stein's avatar
William Stein committed
2133
        for entry in entries:
2134 2135 2136 2137 2138
            if not used_only or entry.used:
                if entry.xdecref_cleanup:
                    self.put_var_xdecref(entry)
                else:
                    self.put_var_decref(entry)
2139

William Stein's avatar
William Stein committed
2140 2141 2142
    def put_var_xdecrefs(self, entries):
        for entry in entries:
            self.put_var_xdecref(entry)
2143

William Stein's avatar
William Stein committed
2144 2145 2146
    def put_var_xdecrefs_clear(self, entries):
        for entry in entries:
            self.put_var_xdecref_clear(entry)
2147

2148
    def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
2149
        from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
2150
        self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
2151 2152 2153
        self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
2154
        from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
2155
        self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
2156 2157 2158 2159 2160
        self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    def put_xgiveref_memoryviewslice(self, slice_cname):
        self.put_xgiveref("%s.memview" % slice_cname)

2161
    def put_init_to_py_none(self, cname, type, nanny=True):
2162
        from .PyrexTypes import py_object_type, typecast
William Stein's avatar
William Stein committed
2163
        py_none = typecast(type, py_object_type, "Py_None")
2164 2165 2166 2167
        if nanny:
            self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))
        else:
            self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))
2168

2169
    def put_init_var_to_py_none(self, entry, template = "%s", nanny=True):
William Stein's avatar
William Stein committed
2170 2171
        code = template % entry.cname
        #if entry.type.is_extension_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
2172
        #    code = "((PyObject*)%s)" % code
2173
        self.put_init_to_py_none(code, entry.type, nanny)
2174
        if entry.in_closure:
Stefan Behnel's avatar
Stefan Behnel committed
2175
            self.put_giveref('Py_None')
William Stein's avatar
William Stein committed
2176

2177
    def put_pymethoddef(self, entry, term, allow_skip=True):
2178
        if entry.is_special or entry.name == '__getattribute__':
2179
            if entry.name not in special_py_methods:
2180 2181
                if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']:
                    pass
2182 2183 2184
                # Python's typeobject.c will automatically fill in our slot
                # in add_operators() (called by PyType_Ready) with a value
                # that's better than ours.
2185
                elif allow_skip:
2186
                    return
2187
        from .TypeSlots import method_coexist
William Stein's avatar
William Stein committed
2188 2189 2190 2191
        if entry.doc:
            doc_code = entry.doc_cname
        else:
            doc_code = 0
2192 2193
        method_flags = entry.signature.method_flags()
        if method_flags:
2194
            if entry.is_special:
2195
                method_flags += [method_coexist]
2196
            func_ptr = entry.func_cname
Stefan Behnel's avatar
Stefan Behnel committed
2197
            # Add required casts, but try not to shadow real warnings.
2198 2199 2200 2201 2202 2203
            if 'METH_NOARGS' in method_flags:
                cast = 'PyNoArgsFunction'
            else:
                cast = '__Pyx_PyCFunctionFast' if 'METH_FASTCALL' in method_flags else 'PyCFunction'
                if 'METH_KEYWORDS' in method_flags:
                    cast += 'WithKeywords'
2204 2205
            if cast != 'PyCFunction':
                func_ptr = '(void*)(%s)%s' % (cast, func_ptr)
2206
            self.putln(
2207
                '{"%s", (PyCFunction)%s, %s, %s}%s' % (
2208
                    entry.name,
2209
                    func_ptr,
2210
                    "|".join(method_flags),
2211 2212
                    doc_code,
                    term))
2213

2214 2215
    # GIL methods

2216
    def put_ensure_gil(self, declare_gilstate=True, variable=None):
2217 2218 2219 2220 2221 2222
        """
        Acquire the GIL. The generated code is safe even when no PyThreadState
        has been allocated for this thread (for threads not initialized by
        using the Python API). Additionally, the code generated by this method
        may be called recursively.
        """
2223 2224
        self.globalstate.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
2225 2226 2227 2228
        if self.globalstate.directives['fast_gil']:
          self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c"))
        else:
          self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c"))
2229
        self.putln("#ifdef WITH_THREAD")
2230 2231 2232 2233
        if not variable:
            variable = '__pyx_gilstate_save'
            if declare_gilstate:
                self.put("PyGILState_STATE ")
2234
        self.putln("%s = __Pyx_PyGILState_Ensure();" % variable)
2235 2236
        self.putln("#endif")

2237
    def put_release_ensured_gil(self, variable=None):
2238 2239 2240
        """
        Releases the GIL, corresponds to `put_ensure_gil`.
        """
2241 2242 2243 2244
        if self.globalstate.directives['fast_gil']:
          self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c"))
        else:
          self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c"))
2245 2246
        if not variable:
            variable = '__pyx_gilstate_save'
2247
        self.putln("#ifdef WITH_THREAD")
2248
        self.putln("__Pyx_PyGILState_Release(%s);" % variable)
2249 2250
        self.putln("#endif")

2251
    def put_acquire_gil(self, variable=None):
2252 2253 2254 2255
        """
        Acquire the GIL. The thread's thread state must have been initialized
        by a previous `put_release_gil`
        """
2256 2257 2258 2259
        if self.globalstate.directives['fast_gil']:
          self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c"))
        else:
          self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c"))
2260
        self.putln("#ifdef WITH_THREAD")
2261
        self.putln("__Pyx_FastGIL_Forget();")
2262 2263
        if variable:
            self.putln('_save = %s;' % variable)
2264
        self.putln("Py_BLOCK_THREADS")
2265
        self.putln("#endif")
2266

2267
    def put_release_gil(self, variable=None):
2268
        "Release the GIL, corresponds to `put_acquire_gil`."
2269 2270 2271 2272
        if self.globalstate.directives['fast_gil']:
          self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c"))
        else:
          self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c"))
2273
        self.putln("#ifdef WITH_THREAD")
2274
        self.putln("PyThreadState *_save;")
2275
        self.putln("Py_UNBLOCK_THREADS")
2276 2277
        if variable:
            self.putln('%s = _save;' % variable)
2278
        self.putln("__Pyx_FastGIL_Remember();")
2279
        self.putln("#endif")
2280

2281 2282
    def declare_gilstate(self):
        self.putln("#ifdef WITH_THREAD")
2283
        self.putln("PyGILState_STATE __pyx_gilstate_save;")
2284 2285
        self.putln("#endif")

2286 2287
    # error handling

Robert Bradshaw's avatar
Robert Bradshaw committed
2288 2289 2290 2291
    def put_error_if_neg(self, pos, value):
#        return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))  # TODO this path is almost _never_ taken, yet this macro makes is slower!
        return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))

2292
    def put_error_if_unbound(self, pos, entry, in_nogil_context=False):
2293
        from . import ExprNodes
2294
        if entry.from_closure:
2295 2296 2297
            func = '__Pyx_RaiseClosureNameError'
            self.globalstate.use_utility_code(
                ExprNodes.raise_closure_name_error_utility_code)
2298 2299 2300 2301
        elif entry.type.is_memoryviewslice and in_nogil_context:
            func = '__Pyx_RaiseUnboundMemoryviewSliceNogil'
            self.globalstate.use_utility_code(
                ExprNodes.raise_unbound_memoryview_utility_code_nogil)
2302
        else:
2303 2304 2305
            func = '__Pyx_RaiseUnboundLocalError'
            self.globalstate.use_utility_code(
                ExprNodes.raise_unbound_local_error_utility_code)
Mark Florisson's avatar
Mark Florisson committed
2306

2307
        self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % (
Mark Florisson's avatar
Mark Florisson committed
2308 2309 2310 2311
                                entry.type.check_for_null_code(entry.cname),
                                func,
                                entry.name,
                                self.error_goto(pos)))
2312

2313
    def set_error_info(self, pos, used=False):
2314
        self.funcstate.should_declare_error_indicator = True
2315 2316
        if used:
            self.funcstate.uses_error_indicator = True
2317
        if self.code_config.c_line_in_traceback:
2318
            cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
Robert Bradshaw's avatar
Robert Bradshaw committed
2319 2320
        else:
            cinfo = ""
2321

2322
        return "%s = %s[%s]; %s = %s;%s" % (
William Stein's avatar
William Stein committed
2323 2324 2325 2326 2327
            Naming.filename_cname,
            Naming.filetable_cname,
            self.lookup_filename(pos[0]),
            Naming.lineno_cname,
            pos[1],
2328
            cinfo)
2329

2330 2331 2332
    def error_goto(self, pos):
        lbl = self.funcstate.error_label
        self.funcstate.use_label(lbl)
2333 2334 2335
        return "__PYX_ERR(%s, %s, %s)" % (
            self.lookup_filename(pos[0]),
            pos[1],
2336
            lbl)
2337 2338 2339

    def error_goto_if(self, cond, pos):
        return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos))
2340

Robert Bradshaw's avatar
Robert Bradshaw committed
2341 2342
    def error_goto_if_null(self, cname, pos):
        return self.error_goto_if("!%s" % cname, pos)
2343

Robert Bradshaw's avatar
Robert Bradshaw committed
2344 2345
    def error_goto_if_neg(self, cname, pos):
        return self.error_goto_if("%s < 0" % cname, pos)
2346

Robert Bradshaw's avatar
Robert Bradshaw committed
2347 2348
    def error_goto_if_PyErr(self, pos):
        return self.error_goto_if("PyErr_Occurred()", pos)
2349

William Stein's avatar
William Stein committed
2350
    def lookup_filename(self, filename):
2351
        return self.globalstate.lookup_filename(filename)
William Stein's avatar
William Stein committed
2352

2353
    def put_declare_refcount_context(self):
2354
        self.putln('__Pyx_RefNannyDeclarations')
2355

2356 2357
    def put_setup_refcount_context(self, name, acquire_gil=False):
        if acquire_gil:
2358 2359
            self.globalstate.use_utility_code(
                UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
2360
        self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0))
2361

2362
    def put_finish_refcount_context(self):
2363
        self.putln("__Pyx_RefNannyFinishContext();")
2364

2365
    def put_add_traceback(self, qualified_name, include_cline=True):
2366 2367 2368
        """
        Build a Python traceback for propagating exceptions.

2369
        qualified_name should be the qualified name of the function.
2370 2371 2372
        """
        format_tuple = (
            qualified_name,
2373
            Naming.clineno_cname if include_cline else 0,
2374 2375 2376
            Naming.lineno_cname,
            Naming.filename_cname,
        )
2377
        self.funcstate.uses_error_indicator = True
2378 2379
        self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple)

2380
    def put_unraisable(self, qualified_name, nogil=False):
2381 2382 2383 2384 2385 2386 2387 2388 2389 2390
        """
        Generate code to print a Python warning for an unraisable exception.

        qualified_name should be the qualified name of the function.
        """
        format_tuple = (
            qualified_name,
            Naming.clineno_cname,
            Naming.lineno_cname,
            Naming.filename_cname,
2391 2392
            self.globalstate.directives['unraisable_tracebacks'],
            nogil,
2393 2394
        )
        self.funcstate.uses_error_indicator = True
2395
        self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %d, %d);' % format_tuple)
2396 2397 2398
        self.globalstate.use_utility_code(
            UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))

2399 2400 2401 2402
    def put_trace_declarations(self):
        self.putln('__Pyx_TraceDeclarations')

    def put_trace_frame_init(self, codeobj=None):
2403 2404
        if codeobj:
            self.putln('__Pyx_TraceFrameInit(%s)' % codeobj)
2405

2406
    def put_trace_call(self, name, pos, nogil=False):
2407 2408
        self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d, %s);' % (
            name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil, self.error_goto(pos)))
2409

Robert Bradshaw's avatar
Robert Bradshaw committed
2410 2411
    def put_trace_exception(self):
        self.putln("__Pyx_TraceException();")
2412

2413 2414
    def put_trace_return(self, retvalue_cname, nogil=False):
        self.putln("__Pyx_TraceReturn(%s, %d);" % (retvalue_cname, nogil))
Robert Bradshaw's avatar
Robert Bradshaw committed
2415

Mark Florisson's avatar
Mark Florisson committed
2416 2417 2418
    def putln_openmp(self, string):
        self.putln("#ifdef _OPENMP")
        self.putln(string)
2419
        self.putln("#endif /* _OPENMP */")
William Stein's avatar
William Stein committed
2420

2421 2422 2423 2424 2425 2426 2427 2428 2429 2430 2431 2432 2433 2434 2435 2436 2437 2438 2439 2440
    def undef_builtin_expect(self, cond):
        """
        Redefine the macros likely() and unlikely to no-ops, depending on
        condition 'cond'
        """
        self.putln("#if %s" % cond)
        self.putln("    #undef likely")
        self.putln("    #undef unlikely")
        self.putln("    #define likely(x)   (x)")
        self.putln("    #define unlikely(x) (x)")
        self.putln("#endif")

    def redef_builtin_expect(self, cond):
        self.putln("#if %s" % cond)
        self.putln("    #undef likely")
        self.putln("    #undef unlikely")
        self.putln("    #define likely(x)   __builtin_expect(!!(x), 1)")
        self.putln("    #define unlikely(x) __builtin_expect(!!(x), 0)")
        self.putln("#endif")

2441

2442
class PyrexCodeWriter(object):
William Stein's avatar
William Stein committed
2443 2444 2445 2446
    # f                file      output file
    # level            int       indentation level

    def __init__(self, outfile_name):
2447
        self.f = Utils.open_new_file(outfile_name)
William Stein's avatar
William Stein committed
2448
        self.level = 0
2449

William Stein's avatar
William Stein committed
2450 2451
    def putln(self, code):
        self.f.write("%s%s\n" % (" " * self.level, code))
2452

William Stein's avatar
William Stein committed
2453 2454
    def indent(self):
        self.level += 1
2455

William Stein's avatar
William Stein committed
2456 2457 2458
    def dedent(self):
        self.level -= 1

2459 2460
class PyxCodeWriter(object):
    """
2461 2462 2463
    Can be used for writing out some Cython code. To use the indenter
    functionality, the Cython.Compiler.Importer module will have to be used
    to load the code to support python 2.4
2464 2465
    """

2466
    def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'):
2467 2468 2469
        self.buffer = buffer or StringIOTree()
        self.level = indent_level
        self.context = context
2470
        self.encoding = encoding
2471 2472 2473

    def indent(self, levels=1):
        self.level += levels
2474
        return True
2475 2476 2477 2478 2479 2480

    def dedent(self, levels=1):
        self.level -= levels

    def indenter(self, line):
        """
2481 2482 2483 2484 2485 2486 2487 2488 2489 2490
        Instead of

            with pyx_code.indenter("for i in range(10):"):
                pyx_code.putln("print i")

        write

            if pyx_code.indenter("for i in range(10);"):
                pyx_code.putln("print i")
                pyx_code.dedent()
2491 2492
        """
        self.putln(line)
2493 2494
        self.indent()
        return True
2495 2496

    def getvalue(self):
2497
        result = self.buffer.getvalue()
2498
        if isinstance(result, bytes):
2499 2500
            result = result.decode(self.encoding)
        return result
2501 2502 2503 2504 2505 2506 2507 2508 2509 2510 2511 2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526

    def putln(self, line, context=None):
        context = context or self.context
        if context:
            line = sub_tempita(line, context)
        self._putln(line)

    def _putln(self, line):
        self.buffer.write("%s%s\n" % (self.level * "    ", line))

    def put_chunk(self, chunk, context=None):
        context = context or self.context
        if context:
            chunk = sub_tempita(chunk, context)

        chunk = textwrap.dedent(chunk)
        for line in chunk.splitlines():
            self._putln(line)

    def insertion_point(self):
        return PyxCodeWriter(self.buffer.insertion_point(), self.level,
                             self.context)

    def named_insertion_point(self, name):
        setattr(self, name, self.insertion_point())

2527 2528

class ClosureTempAllocator(object):
2529
    def __init__(self, klass):
2530 2531 2532 2533 2534 2535 2536 2537 2538 2539
        self.klass = klass
        self.temps_allocated = {}
        self.temps_free = {}
        self.temps_count = 0

    def reset(self):
        for type, cnames in self.temps_allocated.items():
            self.temps_free[type] = list(cnames)

    def allocate_temp(self, type):
2540
        if type not in self.temps_allocated:
2541 2542 2543 2544 2545 2546 2547 2548 2549
            self.temps_allocated[type] = []
            self.temps_free[type] = []
        elif self.temps_free[type]:
            return self.temps_free[type].pop(0)
        cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count)
        self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True)
        self.temps_allocated[type].append(cname)
        self.temps_count += 1
        return cname