Code.py 93.1 KB
Newer Older
1
# cython: language_level=3str
2
# cython: auto_pickle=False
William Stein's avatar
William Stein committed
3
#
4
#   Code output module
William Stein's avatar
William Stein committed
5 6
#

7 8
from __future__ import absolute_import

Stefan Behnel's avatar
Stefan Behnel committed
9
import cython
Stefan Behnel's avatar
Stefan Behnel committed
10 11
cython.declare(os=object, re=object, operator=object, textwrap=object,
               Template=object, Naming=object, Options=object, StringEncoding=object,
Stefan Behnel's avatar
Stefan Behnel committed
12
               Utils=object, SourceDescriptor=object, StringIOTree=object,
Stefan Behnel's avatar
Stefan Behnel committed
13 14
               DebugFlags=object, basestring=object, defaultdict=object,
               closing=object, partial=object)
Stefan Behnel's avatar
Stefan Behnel committed
15

16 17
import hashlib
import operator
18
import os
19
import re
20
import shutil
21
import textwrap
22 23
from string import Template
from functools import partial
Stefan Behnel's avatar
Stefan Behnel committed
24 25
from contextlib import closing
from collections import defaultdict
26

27 28 29 30
from . import Naming
from . import Options
from . import DebugFlags
from . import StringEncoding
31
from . import Version
32 33 34
from .. import Utils
from .Scanning import SourceDescriptor
from ..StringIOTree import StringIOTree
35

Stefan Behnel's avatar
Stefan Behnel committed
36
try:
Stefan Behnel's avatar
Stefan Behnel committed
37 38 39
    from __builtin__ import basestring
except ImportError:
    from builtins import str as basestring
40

41 42

non_portable_builtins_map = {
43
    # builtins that have different names in different Python versions
44 45
    'bytes'         : ('PY_MAJOR_VERSION < 3',  'str'),
    'unicode'       : ('PY_MAJOR_VERSION >= 3', 'str'),
46
    'basestring'    : ('PY_MAJOR_VERSION >= 3', 'str'),
47
    'xrange'        : ('PY_MAJOR_VERSION >= 3', 'range'),
48
    'raw_input'     : ('PY_MAJOR_VERSION >= 3', 'input'),
49
}
50

51 52 53 54 55 56 57 58 59 60
ctypedef_builtins_map = {
    # types of builtins in "ctypedef class" statements which we don't
    # import either because the names conflict with C types or because
    # the type simply is not exposed.
    'py_int'             : '&PyInt_Type',
    'py_long'            : '&PyLong_Type',
    'py_float'           : '&PyFloat_Type',
    'wrapper_descriptor' : '&PyWrapperDescr_Type',
}

61 62
basicsize_builtins_map = {
    # builtins whose type has a different tp_basicsize than sizeof(...)
63 64
    'PyTypeObject': 'PyHeapTypeObject',
}
65

66
uncachable_builtins = [
67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99
    # Global/builtin names that cannot be cached because they may or may not
    # be available at import time, for various reasons:
    ## - Py3.7+
    'breakpoint',  # might deserve an implementation in Cython
    ## - Py3.4+
    '__loader__',
    '__spec__',
    ## - Py3+
    'BlockingIOError',
    'BrokenPipeError',
    'ChildProcessError',
    'ConnectionAbortedError',
    'ConnectionError',
    'ConnectionRefusedError',
    'ConnectionResetError',
    'FileExistsError',
    'FileNotFoundError',
    'InterruptedError',
    'IsADirectoryError',
    'ModuleNotFoundError',
    'NotADirectoryError',
    'PermissionError',
    'ProcessLookupError',
    'RecursionError',
    'ResourceWarning',
    #'StopAsyncIteration',  # backported
    'TimeoutError',
    '__build_class__',
    'ascii',  # might deserve an implementation in Cython
    #'exec',  # implemented in Cython
    ## - Py2.7+
    'memoryview',
    ## - platform specific
100
    'WindowsError',
101 102
    ## - others
    '_',  # e.g. used by gettext
103
]
104

105 106 107 108 109 110 111
special_py_methods = set([
    '__cinit__', '__dealloc__', '__richcmp__', '__next__',
    '__await__', '__aiter__', '__anext__',
    '__getreadbuffer__', '__getwritebuffer__', '__getsegcount__',
    '__getcharbuffer__', '__getbuffer__', '__releasebuffer__'
])

112 113 114 115
modifier_output_mapper = {
    'inline': 'CYTHON_INLINE'
}.get

Stefan Behnel's avatar
Stefan Behnel committed
116

117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159
class IncludeCode(object):
    """
    An include file and/or verbatim C code to be included in the
    generated sources.
    """
    # attributes:
    #
    #  pieces    {order: unicode}: pieces of C code to be generated.
    #            For the included file, the key "order" is zero.
    #            For verbatim include code, the "order" is the "order"
    #            attribute of the original IncludeCode where this piece
    #            of C code was first added. This is needed to prevent
    #            duplication if the same include code is found through
    #            multiple cimports.
    #  location  int: where to put this include in the C sources, one
    #            of the constants INITIAL, EARLY, LATE
    #  order     int: sorting order (automatically set by increasing counter)

    # Constants for location. If the same include occurs with different
    # locations, the earliest one takes precedense.
    INITIAL = 0
    EARLY = 1
    LATE = 2

    counter = 1   # Counter for "order"

    def __init__(self, include=None, verbatim=None, late=True, initial=False):
        self.order = self.counter
        type(self).counter += 1
        self.pieces = {}

        if include:
            if include[0] == '<' and include[-1] == '>':
                self.pieces[0] = u'#include {0}'.format(include)
                late = False  # system include is never late
            else:
                self.pieces[0] = u'#include "{0}"'.format(include)

        if verbatim:
            self.pieces[self.order] = verbatim

        if initial:
            self.location = self.INITIAL
160
        elif late:
161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192
            self.location = self.LATE
        else:
            self.location = self.EARLY

    def dict_update(self, d, key):
        """
        Insert `self` in dict `d` with key `key`. If that key already
        exists, update the attributes of the existing value with `self`.
        """
        if key in d:
            other = d[key]
            other.location = min(self.location, other.location)
            other.pieces.update(self.pieces)
        else:
            d[key] = self

    def sortkey(self):
        return self.order

    def mainpiece(self):
        """
        Return the main piece of C code, corresponding to the include
        file. If there was no include file, return None.
        """
        return self.pieces.get(0)

    def write(self, code):
        # Write values of self.pieces dict, sorted by the keys
        for k in sorted(self.pieces):
            code.putln(self.pieces[k])


193 194 195 196 197
def get_utility_dir():
    # make this a function and not global variables:
    # http://trac.cython.org/cython_trac/ticket/475
    Cython_dir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
    return os.path.join(Cython_dir, "Utility")
Mark Florisson's avatar
Mark Florisson committed
198

Stefan Behnel's avatar
Stefan Behnel committed
199

200
class UtilityCodeBase(object):
201 202 203 204 205 206
    """
    Support for loading utility code from a file.

    Code sections in the file can be specified as follows:

        ##### MyUtility.proto #####
207

208
        [proto declarations]
209

210
        ##### MyUtility.init #####
211

212
        [code run at module initialization]
213 214 215

        ##### MyUtility #####
        #@requires: MyOtherUtility
216
        #@substitute: naming
217

218
        [definitions]
219 220 221 222 223 224 225 226 227

    for prototypes and implementation respectively.  For non-python or
    -cython files backslashes should be used instead.  5 to 30 comment
    characters may be used on either side.

    If the @cname decorator is not used and this is a CythonUtilityCode,
    one should pass in the 'name' keyword argument to be used for name
    mangling of such entries.
    """
228 229 230 231

    is_cython_utility = False
    _utility_cache = {}

232
    @classmethod
233
    def _add_utility(cls, utility, type, lines, begin_lineno, tags=None):
234 235
        if utility is None:
            return
236

237 238 239 240 241
        code = '\n'.join(lines)
        if tags and 'substitute' in tags and tags['substitute'] == set(['naming']):
            del tags['substitute']
            try:
                code = Template(code).substitute(vars(Naming))
242
            except (KeyError, ValueError) as e:
243 244 245 246 247
                raise RuntimeError("Error parsing templated utility code of type '%s' at line %d: %s" % (
                    type, begin_lineno, e))

        # remember correct line numbers at least until after templating
        code = '\n' * begin_lineno + code
248

249
        if type == 'proto':
250
            utility[0] = code
251
        elif type == 'impl':
252
            utility[1] = code
253
        else:
254
            all_tags = utility[2]
255
            all_tags[type] = code
256

257
        if tags:
258
            all_tags = utility[2]
Stefan Behnel's avatar
Stefan Behnel committed
259
            for name, values in tags.items():
260
                all_tags.setdefault(name, set()).update(values)
261

262
    @classmethod
263 264 265 266 267
    def load_utilities_from_file(cls, path):
        utilities = cls._utility_cache.get(path)
        if utilities:
            return utilities

268
        filename = os.path.join(get_utility_dir(), path)
269 270 271
        _, ext = os.path.splitext(path)
        if ext in ('.pyx', '.py', '.pxd', '.pxi'):
            comment = '#'
Stefan Behnel's avatar
Stefan Behnel committed
272
            strip_comments = partial(re.compile(r'^\s*#.*').sub, '')
Stefan Behnel's avatar
Stefan Behnel committed
273
            rstrip = StringEncoding._unicode.rstrip
274
        else:
275
            comment = '/'
Stefan Behnel's avatar
Stefan Behnel committed
276
            strip_comments = partial(re.compile(r'^\s*//.*|/\*[^*]*\*/').sub, '')
277
            rstrip = partial(re.compile(r'\s+(\\?)$').sub, r'\1')
278
        match_special = re.compile(
279
            (r'^%(C)s{5,30}\s*(?P<name>(?:\w|\.)+)\s*%(C)s{5,30}|'
Stefan Behnel's avatar
Stefan Behnel committed
280 281
             r'^%(C)s+@(?P<tag>\w+)\s*:\s*(?P<value>(?:\w|[.:])+)') %
            {'C': comment}).match
282
        match_type = re.compile(r'(.+)[.](proto(?:[.]\S+)?|impl|init|cleanup)$').match
283

Stefan Behnel's avatar
Stefan Behnel committed
284
        with closing(Utils.open_source_file(filename, encoding='UTF-8')) as f:
285
            all_lines = f.readlines()
286

287
        utilities = defaultdict(lambda: [None, None, {}])
288
        lines = []
Stefan Behnel's avatar
Stefan Behnel committed
289
        tags = defaultdict(set)
290 291 292
        utility = type = None
        begin_lineno = 0

Mark Florisson's avatar
Mark Florisson committed
293
        for lineno, line in enumerate(all_lines):
294
            m = match_special(line)
Mark Florisson's avatar
Mark Florisson committed
295
            if m:
296
                if m.group('name'):
297 298 299 300 301 302
                    cls._add_utility(utility, type, lines, begin_lineno, tags)

                    begin_lineno = lineno + 1
                    del lines[:]
                    tags.clear()

303 304 305 306
                    name = m.group('name')
                    mtype = match_type(name)
                    if mtype:
                        name, type = mtype.groups()
307
                    else:
308
                        type = 'impl'
Stefan Behnel's avatar
Stefan Behnel committed
309
                    utility = utilities[name]
Mark Florisson's avatar
Mark Florisson committed
310
                else:
Stefan Behnel's avatar
Stefan Behnel committed
311 312
                    tags[m.group('tag')].add(m.group('value'))
                    lines.append('')  # keep line number correct
Mark Florisson's avatar
Mark Florisson committed
313
            else:
Stefan Behnel's avatar
Stefan Behnel committed
314
                lines.append(rstrip(strip_comments(line)))
Mark Florisson's avatar
Mark Florisson committed
315

316
        if utility is None:
317 318 319
            raise ValueError("Empty utility code file")

        # Don't forget to add the last utility code
320
        cls._add_utility(utility, type, lines, begin_lineno, tags)
321

Stefan Behnel's avatar
Stefan Behnel committed
322
        utilities = dict(utilities)  # un-defaultdict-ify
323 324 325
        cls._utility_cache[path] = utilities
        return utilities

326
    @classmethod
327
    def load(cls, util_code_name, from_file=None, **kwargs):
328
        """
329 330 331 332
        Load utility code from a file specified by from_file (relative to
        Cython/Utility) and name util_code_name.  If from_file is not given,
        load it from the file util_code_name.*.  There should be only one
        file matched by this pattern.
333
        """
334 335 336
        if '::' in util_code_name:
            from_file, util_code_name = util_code_name.rsplit('::', 1)
        if not from_file:
337 338
            utility_dir = get_utility_dir()
            prefix = util_code_name + '.'
339 340 341 342 343 344 345 346 347
            try:
                listing = os.listdir(utility_dir)
            except OSError:
                # XXX the code below assumes as 'zipimport.zipimporter' instance
                # XXX should be easy to generalize, but too lazy right now to write it
                import zipfile
                global __loader__
                loader = __loader__
                archive = loader.archive
Stefan Behnel's avatar
Stefan Behnel committed
348
                with closing(zipfile.ZipFile(archive)) as fileobj:
349 350 351 352 353
                    listing = [os.path.basename(name)
                               for name in fileobj.namelist()
                               if os.path.join(archive, name).startswith(utility_dir)]
            files = [filename for filename in listing
                     if filename.startswith(prefix)]
354 355 356 357 358
            if not files:
                raise ValueError("No match found for utility code " + util_code_name)
            if len(files) > 1:
                raise ValueError("More than one filename match found for utility code " + util_code_name)
            from_file = files[0]
359 360

        utilities = cls.load_utilities_from_file(from_file)
361
        proto, impl, tags = utilities[util_code_name]
362 363 364

        if tags:
            orig_kwargs = kwargs.copy()
Stefan Behnel's avatar
Stefan Behnel committed
365
            for name, values in tags.items():
366 367 368 369
                if name in kwargs:
                    continue
                # only pass lists when we have to: most argument expect one value or None
                if name == 'requires':
370 371 372 373 374 375 376
                    if orig_kwargs:
                        values = [cls.load(dep, from_file, **orig_kwargs)
                                  for dep in sorted(values)]
                    else:
                        # dependencies are rarely unique, so use load_cached() when we can
                        values = [cls.load_cached(dep, from_file)
                                  for dep in sorted(values)]
377 378 379
                elif not values:
                    values = None
                elif len(values) == 1:
380
                    values = list(values)[0]
381
                kwargs[name] = values
Mark Florisson's avatar
Mark Florisson committed
382

383 384 385 386
        if proto is not None:
            kwargs['proto'] = proto
        if impl is not None:
            kwargs['impl'] = impl
Mark Florisson's avatar
Mark Florisson committed
387

388 389
        if 'name' not in kwargs:
            kwargs['name'] = util_code_name
Mark Florisson's avatar
Mark Florisson committed
390

391 392 393 394 395
        if 'file' not in kwargs and from_file:
            kwargs['file'] = from_file
        return cls(**kwargs)

    @classmethod
396
    def load_cached(cls, utility_code_name, from_file=None, __cache={}):
397
        """
398
        Calls .load(), but using a per-type cache based on utility name and file name.
399
        """
400 401
        key = (cls, from_file, utility_code_name)
        try:
402
            return __cache[key]
403 404
        except KeyError:
            pass
405
        code = __cache[key] = cls.load(utility_code_name, from_file)
406
        return code
407

408 409 410 411 412 413 414
    @classmethod
    def load_as_string(cls, util_code_name, from_file=None, **kwargs):
        """
        Load a utility code as a string. Returns (proto, implementation)
        """
        util = cls.load(util_code_name, from_file, **kwargs)
        proto, impl = util.proto, util.impl
415
        return util.format_code(proto), util.format_code(impl)
416

417
    def format_code(self, code_string, replace_empty_lines=re.compile(r'\n\n+').sub):
418 419 420 421 422 423 424
        """
        Format a code section for output.
        """
        if code_string:
            code_string = replace_empty_lines('\n', code_string.strip()) + '\n\n'
        return code_string

425
    def __str__(self):
426
        return "<%s(%s)>" % (type(self).__name__, self.name)
427

428
    def get_tree(self, **kwargs):
429
        pass
430

431 432 433 434
    def __deepcopy__(self, memodict=None):
        # No need to deep-copy utility code since it's essentially immutable.
        return self

Stefan Behnel's avatar
Stefan Behnel committed
435

436
class UtilityCode(UtilityCodeBase):
437 438 439 440 441 442 443 444
    """
    Stores utility code to add during code generation.

    See GlobalState.put_utility_code.

    hashes/equals by instance

    proto           C prototypes
luz.paz's avatar
luz.paz committed
445
    impl            implementation code
446 447 448 449 450 451 452 453
    init            code to call on module initialization
    requires        utility code dependencies
    proto_block     the place in the resulting file where the prototype should
                    end up
    name            name of the utility code (or None)
    file            filename of the utility code file this utility was loaded
                    from (or None)
    """
454

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
455
    def __init__(self, proto=None, impl=None, init=None, cleanup=None, requires=None,
456
                 proto_block='utility_code_proto', name=None, file=None):
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
457
        # proto_block: Which code block to dump prototype in. See GlobalState.
458 459 460 461 462 463 464
        self.proto = proto
        self.impl = impl
        self.init = init
        self.cleanup = cleanup
        self.requires = requires
        self._cache = {}
        self.specialize_list = []
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
465
        self.proto_block = proto_block
466
        self.name = name
467
        self.file = file
468

469 470 471 472 473 474
    def __hash__(self):
        return hash((self.proto, self.impl))

    def __eq__(self, other):
        if self is other:
            return True
475 476
        self_type, other_type = type(self), type(other)
        if self_type is not other_type and not (isinstance(other, self_type) or isinstance(self, other_type)):
477 478 479 480 481 482 483 484 485 486 487 488 489 490 491
            return False

        self_proto = getattr(self, 'proto', None)
        other_proto = getattr(other, 'proto', None)
        return (self_proto, self.impl) == (other_proto, other.impl)

    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return s % context

    def specialize(self, pyrex_type=None, **data):
492 493
        # Dicts aren't hashable...
        if pyrex_type is not None:
494
            data['type'] = pyrex_type.empty_declaration_code()
Craig Citro's avatar
Craig Citro committed
495
            data['type_name'] = pyrex_type.specialization_name()
Stefan Behnel's avatar
Stefan Behnel committed
496
        key = tuple(sorted(data.items()))
497 498 499 500 501 502 503
        try:
            return self._cache[key]
        except KeyError:
            if self.requires is None:
                requires = None
            else:
                requires = [r.specialize(data) for r in self.requires]
504

505
            s = self._cache[key] = UtilityCode(
Stefan Behnel's avatar
Stefan Behnel committed
506 507 508 509 510 511
                self.none_or_sub(self.proto, data),
                self.none_or_sub(self.impl, data),
                self.none_or_sub(self.init, data),
                self.none_or_sub(self.cleanup, data),
                requires,
                self.proto_block)
512

513 514 515
            self.specialize_list.append(s)
            return s

516 517 518
    def inject_string_constants(self, impl, output):
        """Replace 'PYIDENT("xyz")' by a constant Python identifier cname.
        """
519
        if 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl:
520 521
            return False, impl

522 523
        replacements = {}
        def externalise(matchobj):
524
            key = matchobj.groups()
525
            try:
526
                cname = replacements[key]
527
            except KeyError:
528 529 530
                str_type, name = key
                cname = replacements[key] = output.get_py_string_const(
                        StringEncoding.EncodedString(name), identifier=str_type == 'IDENT').cname
531 532
            return cname

533 534
        impl = re.sub(r'PY(IDENT|UNICODE)\("([^"]+)"\)', externalise, impl)
        assert 'PYIDENT(' not in impl and 'PYUNICODE(' not in impl
535
        return bool(replacements), impl
536

537 538 539 540 541 542
    def inject_unbound_methods(self, impl, output):
        """Replace 'UNBOUND_METHOD(type, "name")' by a constant Python identifier cname.
        """
        if 'CALL_UNBOUND_METHOD(' not in impl:
            return False, impl

543
        utility_code = set()
544
        def externalise(matchobj):
545 546 547 548 549 550 551 552 553 554 555 556
            type_cname, method_name, obj_cname, args = matchobj.groups()
            args = [arg.strip() for arg in args[1:].split(',')] if args else []
            assert len(args) < 3, "CALL_UNBOUND_METHOD() does not support %d call arguments" % len(args)
            return output.cached_unbound_method_call_code(obj_cname, type_cname, method_name, args)

        impl = re.sub(
            r'CALL_UNBOUND_METHOD\('
            r'([a-zA-Z_]+),'      # type cname
            r'\s*"([^"]+)",'      # method name
            r'\s*([^),]+)'        # object cname
            r'((?:,\s*[^),]+)*)'  # args*
            r'\)', externalise, impl)
557
        assert 'CALL_UNBOUND_METHOD(' not in impl
558 559 560 561

        for helper in sorted(utility_code):
            output.use_utility_code(UtilityCode.load_cached(helper, "ObjectHandling.c"))
        return bool(utility_code), impl
562

563 564 565 566 567 568 569 570 571 572 573 574
    def wrap_c_strings(self, impl):
        """Replace CSTRING('''xyz''') by a C compatible string
        """
        if 'CSTRING(' not in impl:
            return impl

        def split_string(matchobj):
            content = matchobj.group(1).replace('"', '\042')
            return ''.join(
                '"%s\\n"\n' % line if not line.endswith('\\') or line.endswith('\\\\') else '"%s"\n' % line[:-1]
                for line in content.splitlines())

575
        impl = re.sub(r'CSTRING\(\s*"""([^"]*(?:"[^"]+)*)"""\s*\)', split_string, impl)
576 577 578
        assert 'CSTRING(' not in impl
        return impl

579 580 581 582 583
    def put_code(self, output):
        if self.requires:
            for dependency in self.requires:
                output.use_utility_code(dependency)
        if self.proto:
584 585 586 587
            writer = output[self.proto_block]
            writer.putln("/* %s.proto */" % self.name)
            writer.put_or_include(
                self.format_code(self.proto), '%s_proto' % self.name)
588
        if self.impl:
589
            impl = self.format_code(self.wrap_c_strings(self.impl))
590 591
            is_specialised1, impl = self.inject_string_constants(impl, output)
            is_specialised2, impl = self.inject_unbound_methods(impl, output)
592
            writer = output['utility_code_def']
Stefan Behnel's avatar
Stefan Behnel committed
593
            writer.putln("/* %s */" % self.name)
594
            if not (is_specialised1 or is_specialised2):
595
                # no module specific adaptations => can be reused
596
                writer.put_or_include(impl, '%s_impl' % self.name)
597
            else:
598
                writer.put(impl)
599 600
        if self.init:
            writer = output['init_globals']
601
            writer.putln("/* %s.init */" % self.name)
602
            if isinstance(self.init, basestring):
603
                writer.put(self.format_code(self.init))
604 605
            else:
                self.init(writer, output.module_pos)
606 607
            writer.putln(writer.error_goto_if_PyErr(output.module_pos))
            writer.putln()
608 609
        if self.cleanup and Options.generate_cleanup_code:
            writer = output['cleanup_globals']
610
            writer.putln("/* %s.cleanup */" % self.name)
611
            if isinstance(self.cleanup, basestring):
612 613 614
                writer.put_or_include(
                    self.format_code(self.cleanup),
                    '%s_cleanup' % self.name)
615 616
            else:
                self.cleanup(writer, output.module_pos)
617 618


619
def sub_tempita(s, context, file=None, name=None):
620 621 622
    "Run tempita on string s with given context."
    if not s:
        return None
623

624 625 626 627
    if file:
        context['__name'] = "%s:%s" % (file, name)
    elif name:
        context['__name'] = name
628

629
    from ..Tempita import sub
630
    return sub(s, **context)
631

632

633
class TempitaUtilityCode(UtilityCode):
634
    def __init__(self, name=None, proto=None, impl=None, init=None, file=None, context=None, **kwargs):
635 636
        if context is None:
            context = {}
637 638
        proto = sub_tempita(proto, context, file, name)
        impl = sub_tempita(impl, context, file, name)
639
        init = sub_tempita(init, context, file, name)
640
        super(TempitaUtilityCode, self).__init__(
641
            proto, impl, init=init, name=name, file=file, **kwargs)
642

643 644 645 646 647 648 649 650 651 652 653 654
    @classmethod
    def load_cached(cls, utility_code_name, from_file=None, context=None, __cache={}):
        context_key = tuple(sorted(context.items())) if context else None
        assert hash(context_key) is not None  # raise TypeError if not hashable
        key = (cls, from_file, utility_code_name, context_key)
        try:
            return __cache[key]
        except KeyError:
            pass
        code = __cache[key] = cls.load(utility_code_name, from_file, context=context)
        return code

655 656 657 658 659 660 661
    def none_or_sub(self, s, context):
        """
        Format a string in this utility code with context. If None, do nothing.
        """
        if s is None:
            return None
        return sub_tempita(s, context, self.file, self.name)
662 663 664 665 666 667 668


class LazyUtilityCode(UtilityCodeBase):
    """
    Utility code that calls a callback with the root code writer when
    available. Useful when you only have 'env' but not 'code'.
    """
669
    __name__ = '<lazy>'
670
    requires = None
671 672 673 674 675 676 677 678

    def __init__(self, callback):
        self.callback = callback

    def put_code(self, globalstate):
        utility = self.callback(globalstate.rootwriter)
        globalstate.use_utility_code(utility)

679

680 681 682 683 684 685 686 687 688
class FunctionState(object):
    # return_label     string          function return point label
    # error_label      string          error catch point label
    # continue_label   string          loop continue point label
    # break_label      string          loop break point label
    # return_from_error_cleanup_label string
    # label_counter    integer         counter for naming labels
    # in_try_finally   boolean         inside try of try...finally
    # exc_vars         (string * 3)    exception variables for reraise, or None
689
    # can_trace        boolean         line tracing is supported in the current context
690
    # scope            Scope           the scope object of the current function
691

692
    # Not used for now, perhaps later
693
    def __init__(self, owner, names_taken=set(), scope=None):
694
        self.names_taken = names_taken
695
        self.owner = owner
696
        self.scope = scope
697

698 699
        self.error_label = None
        self.label_counter = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
700
        self.labels_used = set()
701 702 703 704
        self.return_label = self.new_label()
        self.new_error_label()
        self.continue_label = None
        self.break_label = None
705
        self.yield_labels = []
706

707 708
        self.in_try_finally = 0
        self.exc_vars = None
709
        self.current_except = None
710
        self.can_trace = False
711
        self.gil_owned = True
712

713
        self.temps_allocated = [] # of (name, type, manage_ref, static)
714 715
        self.temps_free = {} # (type, manage_ref) -> list of free vars with same type/managed status
        self.temps_used_type = {} # name -> (type, manage_ref)
716
        self.temp_counter = 0
717
        self.closure_temps = None
718

719 720 721 722
        # This is used to collect temporaries, useful to find out which temps
        # need to be privatized in parallel sections
        self.collect_temps_stack = []

723 724 725 726 727
        # This is used for the error indicator, which needs to be local to the
        # function. It used to be global, which relies on the GIL being held.
        # However, exceptions may need to be propagated through 'nogil'
        # sections, in which case we introduce a race condition.
        self.should_declare_error_indicator = False
728
        self.uses_error_indicator = False
729

730 731
    # labels

732
    def new_label(self, name=None):
733 734
        n = self.label_counter
        self.label_counter = n + 1
735 736 737 738
        label = "%s%d" % (Naming.label_prefix, n)
        if name is not None:
            label += '_' + name
        return label
739

740 741
    def new_yield_label(self, expr_type='yield'):
        label = self.new_label('resume_from_%s' % expr_type)
742 743 744 745
        num_and_label = (len(self.yield_labels) + 1, label)
        self.yield_labels.append(num_and_label)
        return num_and_label

746 747
    def new_error_label(self):
        old_err_lbl = self.error_label
748
        self.error_label = self.new_label('error')
749
        return old_err_lbl
750

751 752 753 754
    def get_loop_labels(self):
        return (
            self.continue_label,
            self.break_label)
755

756 757 758
    def set_loop_labels(self, labels):
        (self.continue_label,
         self.break_label) = labels
759

760 761 762
    def new_loop_labels(self):
        old_labels = self.get_loop_labels()
        self.set_loop_labels(
763
            (self.new_label("continue"),
Robert Bradshaw's avatar
Robert Bradshaw committed
764
             self.new_label("break")))
765
        return old_labels
766

767 768 769 770 771 772 773 774 775 776 777 778 779 780 781 782
    def get_all_labels(self):
        return (
            self.continue_label,
            self.break_label,
            self.return_label,
            self.error_label)

    def set_all_labels(self, labels):
        (self.continue_label,
         self.break_label,
         self.return_label,
         self.error_label) = labels

    def all_new_labels(self):
        old_labels = self.get_all_labels()
        new_labels = []
783
        for old_label, name in zip(old_labels, ['continue', 'break', 'return', 'error']):
784
            if old_label:
785
                new_labels.append(self.new_label(name))
786 787 788 789
            else:
                new_labels.append(old_label)
        self.set_all_labels(new_labels)
        return old_labels
790

791
    def use_label(self, lbl):
Stefan Behnel's avatar
Stefan Behnel committed
792
        self.labels_used.add(lbl)
793

794 795 796
    def label_used(self, lbl):
        return lbl in self.labels_used

797 798
    # temp handling

799
    def allocate_temp(self, type, manage_ref, static=False):
800 801 802 803 804
        """
        Allocates a temporary (which may create a new one or get a previously
        allocated and released one of the same type). Type is simply registered
        and handed back, but will usually be a PyrexType.

805 806 807 808 809
        If type.is_pyobject, manage_ref comes into play. If manage_ref is set to
        True, the temp will be decref-ed on return statements and in exception
        handling clauses. Otherwise the caller has to deal with any reference
        counting of the variable.

810 811 812 813
        If not type.is_pyobject, then manage_ref will be ignored, but it
        still has to be passed. It is recommended to pass False by convention
        if it is known that type will never be a Python object.

814 815 816 817
        static=True marks the temporary declaration with "static".
        This is only used when allocating backing store for a module-level
        C array literals.

818 819
        A C string referring to the variable is returned.
        """
820 821
        if type.is_cv_qualified and not type.is_reference:
            type = type.cv_base_type
822
        elif type.is_reference and not type.is_fake_reference:
823
            type = type.ref_base_type
824
        if not type.is_pyobject and not type.is_memoryviewslice:
825 826 827
            # Make manage_ref canonical, so that manage_ref will always mean
            # a decref is needed.
            manage_ref = False
828

829
        freelist = self.temps_free.get((type, manage_ref))
830 831 832
        if freelist is not None and freelist[0]:
            result = freelist[0].pop()
            freelist[1].remove(result)
833
        else:
834 835 836
            while True:
                self.temp_counter += 1
                result = "%s%d" % (Naming.codewriter_temp_prefix, self.temp_counter)
837
                if result not in self.names_taken: break
838
            self.temps_allocated.append((result, type, manage_ref, static))
839
        self.temps_used_type[result] = (type, manage_ref)
840
        if DebugFlags.debug_temp_code_comments:
841
            self.owner.putln("/* %s allocated (%s) */" % (result, type))
842 843 844 845

        if self.collect_temps_stack:
            self.collect_temps_stack[-1].add((result, type))

846 847 848 849 850 851 852
        return result

    def release_temp(self, name):
        """
        Releases a temporary so that it can be reused by other code needing
        a temp of the same type.
        """
853 854
        type, manage_ref = self.temps_used_type[name]
        freelist = self.temps_free.get((type, manage_ref))
855
        if freelist is None:
856
            freelist = ([], set())  # keep order in list and make lookups in set fast
857
            self.temps_free[(type, manage_ref)] = freelist
858
        if name in freelist[1]:
859
            raise RuntimeError("Temp %s freed twice!" % name)
860 861
        freelist[0].append(name)
        freelist[1].add(name)
862 863
        if DebugFlags.debug_temp_code_comments:
            self.owner.putln("/* %s released */" % name)
864

865
    def temps_in_use(self):
866
        """Return a list of (cname,type,manage_ref) tuples of temp names and their type
867 868 869
        that are currently in use.
        """
        used = []
870
        for name, type, manage_ref, static in self.temps_allocated:
871
            freelist = self.temps_free.get((type, manage_ref))
872
            if freelist is None or name not in freelist[1]:
873
                used.append((name, type, manage_ref and type.is_pyobject))
874 875
        return used

876
    def temps_holding_reference(self):
877
        """Return a list of (cname,type) tuples of temp names and their type
878 879
        that are currently in use. This includes only temps of a
        Python object type which owns its reference.
880 881
        """
        return [(name, type)
882
                for name, type, manage_ref in self.temps_in_use()
883
                if manage_ref  and type.is_pyobject]
884 885 886 887 888

    def all_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python objects.
        """
        return [(cname, type)
Stefan Behnel's avatar
Stefan Behnel committed
889 890
                for cname, type, manage_ref, static in self.temps_allocated
                if manage_ref]
891

892 893 894 895 896 897
    def all_free_managed_temps(self):
        """Return a list of (cname, type) tuples of refcount-managed Python
        objects that are not currently in use.  This is used by
        try-except and try-finally blocks to clean up temps in the
        error case.
        """
898 899 900 901 902
        return sorted([  # Enforce deterministic order.
            (cname, type)
            for (type, manage_ref), freelist in self.temps_free.items() if manage_ref
            for cname in freelist[0]
        ])
903

904 905 906 907
    def start_collecting_temps(self):
        """
        Useful to find out which temps were used in a code block
        """
Robert Bradshaw's avatar
Robert Bradshaw committed
908
        self.collect_temps_stack.append(set())
909 910 911

    def stop_collecting_temps(self):
        return self.collect_temps_stack.pop()
912

913 914 915
    def init_closure_temps(self, scope):
        self.closure_temps = ClosureTempAllocator(scope)

916

Stefan Behnel's avatar
Stefan Behnel committed
917 918 919 920 921 922 923
class NumConst(object):
    """Global info about a Python number constant held by GlobalState.

    cname       string
    value       string
    py_type     string     int, long, float
    value_code  string     evaluation code if different from value
924 925
    """

Stefan Behnel's avatar
Stefan Behnel committed
926
    def __init__(self, cname, value, py_type, value_code=None):
927 928
        self.cname = cname
        self.value = value
Stefan Behnel's avatar
Stefan Behnel committed
929 930 931
        self.py_type = py_type
        self.value_code = value_code or value

932

933 934 935 936 937 938 939 940 941 942
class PyObjectConst(object):
    """Global info about a generic constant held by GlobalState.
    """
    # cname       string
    # type        PyrexType

    def __init__(self, cname, type):
        self.cname = cname
        self.type = type

Stefan Behnel's avatar
Stefan Behnel committed
943

944
cython.declare(possible_unicode_identifier=object, possible_bytes_identifier=object,
945
               replace_identifier=object, find_alphanums=object)
946
possible_unicode_identifier = re.compile(br"(?![0-9])\w+$".decode('ascii'), re.U).match
Stefan Behnel's avatar
Stefan Behnel committed
947
possible_bytes_identifier = re.compile(r"(?![0-9])\w+$".encode('ASCII')).match
948
replace_identifier = re.compile(r'[^a-zA-Z0-9_]+').sub
949
find_alphanums = re.compile('([a-zA-Z0-9]+)').findall
950 951 952 953 954 955 956 957 958 959 960 961 962

class StringConst(object):
    """Global info about a C string constant held by GlobalState.
    """
    # cname            string
    # text             EncodedString or BytesLiteral
    # py_strings       {(identifier, encoding) : PyStringConst}

    def __init__(self, cname, text, byte_string):
        self.cname = cname
        self.text = text
        self.escaped_value = StringEncoding.escape_byte_string(byte_string)
        self.py_strings = None
963 964 965
        self.py_versions = []

    def add_py_version(self, version):
966
        if not version:
Stefan Behnel's avatar
Stefan Behnel committed
967
            self.py_versions = [2, 3]
968
        elif version not in self.py_versions:
969
            self.py_versions.append(version)
970

971 972
    def get_py_string_const(self, encoding, identifier=None,
                            is_str=False, py3str_cstring=None):
973 974 975
        py_strings = self.py_strings
        text = self.text

Stefan Behnel's avatar
Stefan Behnel committed
976
        is_str = bool(identifier or is_str)
977
        is_unicode = encoding is None and not is_str
978

979 980 981 982 983 984 985 986 987 988 989 990
        if encoding is None:
            # unicode string
            encoding_key = None
        else:
            # bytes or str
            encoding = encoding.lower()
            if encoding in ('utf8', 'utf-8', 'ascii', 'usascii', 'us-ascii'):
                encoding = None
                encoding_key = None
            else:
                encoding_key = ''.join(find_alphanums(encoding))

991 992 993 994 995 996
        key = (is_str, is_unicode, encoding_key, py3str_cstring)
        if py_strings is not None:
            try:
                return py_strings[key]
            except KeyError:
                pass
997
        else:
998
            self.py_strings = {}
999

1000 1001 1002
        if identifier:
            intern = True
        elif identifier is None:
1003
            if isinstance(text, bytes):
1004
                intern = bool(possible_bytes_identifier(text))
1005 1006
            else:
                intern = bool(possible_unicode_identifier(text))
1007 1008 1009
        else:
            intern = False
        if intern:
1010
            prefix = Naming.interned_prefixes['str']
1011 1012
        else:
            prefix = Naming.py_const_prefix
1013 1014 1015 1016 1017 1018 1019

        if encoding_key:
            encoding_prefix = '_%s' % encoding_key
        else:
            encoding_prefix = ''

        pystring_cname = "%s%s%s_%s" % (
1020 1021
            prefix,
            (is_str and 's') or (is_unicode and 'u') or 'b',
1022
            encoding_prefix,
1023 1024 1025 1026 1027
            self.cname[len(Naming.const_prefix):])

        py_string = PyStringConst(
            pystring_cname, encoding, is_unicode, is_str, py3str_cstring, intern)
        self.py_strings[key] = py_string
1028 1029 1030 1031 1032 1033
        return py_string

class PyStringConst(object):
    """Global info about a Python string constant held by GlobalState.
    """
    # cname       string
1034
    # py3str_cstring string
1035
    # encoding    string
1036
    # intern      boolean
1037 1038
    # is_unicode  boolean
    # is_str      boolean
1039

1040 1041
    def __init__(self, cname, encoding, is_unicode, is_str=False,
                 py3str_cstring=None, intern=False):
1042
        self.cname = cname
1043
        self.py3str_cstring = py3str_cstring
1044 1045 1046
        self.encoding = encoding
        self.is_str = is_str
        self.is_unicode = is_unicode
1047 1048
        self.intern = intern

Stefan Behnel's avatar
Stefan Behnel committed
1049 1050 1051
    def __lt__(self, other):
        return self.cname < other.cname

1052

1053 1054 1055 1056 1057 1058
class GlobalState(object):
    # filename_table   {string : int}  for finding filename table indexes
    # filename_list    [string]        filenames in filename table order
    # input_file_contents dict         contents (=list of lines) of any file that was used as input
    #                                  to create this output C code.  This is
    #                                  used to annotate the comments.
1059
    #
1060
    # utility_codes   set                IDs of used utility code (to avoid reinsertion)
1061 1062 1063 1064 1065 1066
    #
    # declared_cnames  {string:Entry}  used in a transition phase to merge pxd-declared
    #                                  constants etc. into the pyx-declared ones (i.e,
    #                                  check if constants are already added).
    #                                  In time, hopefully the literals etc. will be
    #                                  supplied directly instead.
1067
    #
1068
    # const_cnames_used  dict          global counter for unique constant identifiers
1069
    #
1070

1071 1072
    # parts            {string:CCodeWriter}

1073

1074 1075 1076 1077
    # interned_strings
    # consts
    # interned_nums

1078 1079 1080 1081 1082 1083
    # directives       set             Temporary variable used to track
    #                                  the current set of directives in the code generation
    #                                  process.

    directives = {}

1084 1085
    code_layout = [
        'h_code',
Robert Bradshaw's avatar
Robert Bradshaw committed
1086
        'filename_table',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1087
        'utility_code_proto_before_types',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1088 1089 1090 1091
        'numeric_typedefs',          # Let these detailed individual parts stay!,
        'complex_type_declarations', # as the proper solution is to make a full DAG...
        'type_declarations',         # More coarse-grained blocks would simply hide
        'utility_code_proto',        # the ugliness, not fix it
1092 1093
        'module_declarations',
        'typeinfo',
1094 1095
        'before_global_var',
        'global_var',
1096
        'string_decls',
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1097
        'decls',
Jeroen Demeyer's avatar
Jeroen Demeyer committed
1098
        'late_includes',
1099
        'all_the_rest',
1100 1101
        'pystring_table',
        'cached_builtins',
1102
        'cached_constants',
1103 1104 1105 1106 1107
        'init_globals',
        'init_module',
        'cleanup_globals',
        'cleanup_module',
        'main_method',
1108 1109
        'utility_code_def',
        'end'
1110
    ]
1111

1112

1113
    def __init__(self, writer, module_node, code_config, common_utility_include_dir=None):
1114 1115 1116
        self.filename_table = {}
        self.filename_list = []
        self.input_file_contents = {}
Robert Bradshaw's avatar
Robert Bradshaw committed
1117
        self.utility_codes = set()
1118
        self.declared_cnames = {}
1119
        self.in_utility_code_generation = False
1120
        self.code_config = code_config
1121
        self.common_utility_include_dir = common_utility_include_dir
1122
        self.parts = {}
1123 1124
        self.module_node = module_node # because some utility code generation needs it
                                       # (generating backwards-compatible Get/ReleaseBuffer
1125

1126
        self.const_cnames_used = {}
1127
        self.string_const_index = {}
1128
        self.dedup_const_index = {}
Nikita Nemkin's avatar
Nikita Nemkin committed
1129
        self.pyunicode_ptr_const_index = {}
Stefan Behnel's avatar
Stefan Behnel committed
1130
        self.num_const_index = {}
1131
        self.py_constants = []
1132
        self.cached_cmethods = {}
1133
        self.initialised_constants = set()
1134

1135
        writer.set_global_state(self)
1136
        self.rootwriter = writer
1137

1138 1139 1140 1141
    def initialize_main_c_code(self):
        rootwriter = self.rootwriter
        for part in self.code_layout:
            self.parts[part] = rootwriter.insertion_point()
1142

1143 1144 1145 1146 1147
        if not Options.cache_builtins:
            del self.parts['cached_builtins']
        else:
            w = self.parts['cached_builtins']
            w.enter_cfunc_scope()
1148
            w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedBuiltins(void) {")
1149

1150 1151 1152
        w = self.parts['cached_constants']
        w.enter_cfunc_scope()
        w.putln("")
1153
        w.putln("static CYTHON_SMALL_CODE int __Pyx_InitCachedConstants(void) {")
1154
        w.put_declare_refcount_context()
1155 1156
        w.put_setup_refcount_context("__Pyx_InitCachedConstants")

1157 1158 1159
        w = self.parts['init_globals']
        w.enter_cfunc_scope()
        w.putln("")
1160
        w.putln("static CYTHON_SMALL_CODE int __Pyx_InitGlobals(void) {")
1161

1162 1163 1164 1165 1166 1167
        if not Options.generate_cleanup_code:
            del self.parts['cleanup_globals']
        else:
            w = self.parts['cleanup_globals']
            w.enter_cfunc_scope()
            w.putln("")
1168
            w.putln("static CYTHON_SMALL_CODE void __Pyx_CleanupGlobals(void) {")
1169

1170 1171 1172 1173
        code = self.parts['utility_code_proto']
        code.putln("")
        code.putln("/* --- Runtime support code (head) --- */")

1174
        code = self.parts['utility_code_def']
1175
        if self.code_config.emit_linenums:
1176 1177
            code.write('\n#line 1 "cython_utility"\n')
        code.putln("")
1178
        code.putln("/* --- Runtime support code --- */")
1179

1180
    def finalize_main_c_code(self):
1181 1182 1183 1184 1185 1186
        self.close_global_decls()

        #
        # utility_code_def
        #
        code = self.parts['utility_code_def']
1187 1188
        util = TempitaUtilityCode.load_cached("TypeConversions", "TypeConversion.c")
        code.put(util.format_code(util.impl))
1189 1190
        code.putln("")

1191 1192 1193
    def __getitem__(self, key):
        return self.parts[key]

1194 1195 1196
    #
    # Global constants, interned objects, etc.
    #
1197 1198
    def close_global_decls(self):
        # This is called when it is known that no more global declarations will
1199
        # declared.
1200
        self.generate_const_declarations()
1201
        if Options.cache_builtins:
1202
            w = self.parts['cached_builtins']
1203
            w.putln("return 0;")
1204 1205 1206
            if w.label_used(w.error_label):
                w.put_label(w.error_label)
                w.putln("return -1;")
1207 1208 1209
            w.putln("}")
            w.exit_cfunc_scope()

1210 1211 1212 1213 1214 1215 1216 1217 1218 1219
        w = self.parts['cached_constants']
        w.put_finish_refcount_context()
        w.putln("return 0;")
        if w.label_used(w.error_label):
            w.put_label(w.error_label)
            w.put_finish_refcount_context()
            w.putln("return -1;")
        w.putln("}")
        w.exit_cfunc_scope()

1220
        w = self.parts['init_globals']
1221
        w.putln("return 0;")
1222 1223 1224
        if w.label_used(w.error_label):
            w.put_label(w.error_label)
            w.putln("return -1;")
1225 1226
        w.putln("}")
        w.exit_cfunc_scope()
1227

1228 1229 1230 1231 1232
        if Options.generate_cleanup_code:
            w = self.parts['cleanup_globals']
            w.putln("}")
            w.exit_cfunc_scope()

1233 1234 1235 1236
        if Options.generate_cleanup_code:
            w = self.parts['cleanup_module']
            w.putln("}")
            w.exit_cfunc_scope()
1237

1238
    def put_pyobject_decl(self, entry):
1239
        self['global_var'].putln("static PyObject *%s;" % entry.cname)
1240

1241 1242
    # constant handling at code generation time

1243 1244 1245 1246 1247 1248
    def get_cached_constants_writer(self, target=None):
        if target is not None:
            if target in self.initialised_constants:
                # Return None on second/later calls to prevent duplicate creation code.
                return None
            self.initialised_constants.add(target)
1249 1250
        return self.parts['cached_constants']

1251
    def get_int_const(self, str_value, longness=False):
Stefan Behnel's avatar
Stefan Behnel committed
1252
        py_type = longness and 'long' or 'int'
1253
        try:
Stefan Behnel's avatar
Stefan Behnel committed
1254
            c = self.num_const_index[(str_value, py_type)]
1255
        except KeyError:
Stefan Behnel's avatar
Stefan Behnel committed
1256 1257 1258 1259 1260 1261 1262 1263
            c = self.new_num_const(str_value, py_type)
        return c

    def get_float_const(self, str_value, value_code):
        try:
            c = self.num_const_index[(str_value, 'float')]
        except KeyError:
            c = self.new_num_const(str_value, 'float', value_code)
1264 1265
        return c

1266 1267 1268 1269 1270
    def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None):
        if dedup_key is not None:
            const = self.dedup_const_index.get(dedup_key)
            if const is not None:
                return const
1271
        # create a new Python object constant
Stefan Behnel's avatar
Stefan Behnel committed
1272
        const = self.new_py_const(type, prefix)
1273
        if cleanup_level is not None \
Stefan Behnel's avatar
Stefan Behnel committed
1274
                and cleanup_level <= Options.generate_cleanup_code:
1275
            cleanup_writer = self.parts['cleanup_globals']
1276
            cleanup_writer.putln('Py_CLEAR(%s);' % const.cname)
1277 1278
        if dedup_key is not None:
            self.dedup_const_index[dedup_key] = const
1279
        return const
1280

1281
    def get_string_const(self, text, py_version=None):
1282 1283
        # return a C string constant, creating a new one if necessary
        if text.is_unicode:
1284
            byte_string = text.utf8encode()
1285 1286 1287 1288 1289 1290
        else:
            byte_string = text.byteencode()
        try:
            c = self.string_const_index[byte_string]
        except KeyError:
            c = self.new_string_const(text, byte_string)
1291
        c.add_py_version(py_version)
1292 1293
        return c

1294
    def get_pyunicode_ptr_const(self, text):
1295 1296 1297
        # return a Py_UNICODE[] constant, creating a new one if necessary
        assert text.is_unicode
        try:
Nikita Nemkin's avatar
Nikita Nemkin committed
1298
            c = self.pyunicode_ptr_const_index[text]
1299
        except KeyError:
Nikita Nemkin's avatar
Nikita Nemkin committed
1300
            c = self.pyunicode_ptr_const_index[text] = self.new_const_cname()
1301 1302
        return c

1303 1304
    def get_py_string_const(self, text, identifier=None,
                            is_str=False, unicode_value=None):
1305
        # return a Python string constant, creating a new one if necessary
1306 1307
        py3str_cstring = None
        if is_str and unicode_value is not None \
1308
               and unicode_value.utf8encode() != text.byteencode():
1309 1310 1311 1312
            py3str_cstring = self.get_string_const(unicode_value, py_version=3)
            c_string = self.get_string_const(text, py_version=2)
        else:
            c_string = self.get_string_const(text)
1313 1314
        py_string = c_string.get_py_string_const(
            text.encoding, identifier, is_str, py3str_cstring)
1315 1316
        return py_string

1317 1318 1319
    def get_interned_identifier(self, text):
        return self.get_py_string_const(text, identifier=True)

1320
    def new_string_const(self, text, byte_string):
Stefan Behnel's avatar
Stefan Behnel committed
1321
        cname = self.new_string_const_cname(byte_string)
1322 1323 1324 1325
        c = StringConst(cname, text, byte_string)
        self.string_const_index[byte_string] = c
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1326 1327 1328 1329
    def new_num_const(self, value, py_type, value_code=None):
        cname = self.new_num_const_cname(value, py_type)
        c = NumConst(cname, value, py_type, value_code)
        self.num_const_index[(value, py_type)] = c
1330 1331
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1332 1333
    def new_py_const(self, type, prefix=''):
        cname = self.new_const_cname(prefix)
1334 1335 1336 1337
        c = PyObjectConst(cname, type)
        self.py_constants.append(c)
        return c

Stefan Behnel's avatar
Stefan Behnel committed
1338
    def new_string_const_cname(self, bytes_value):
1339
        # Create a new globally-unique nice name for a C string constant.
Stefan Behnel's avatar
Stefan Behnel committed
1340
        value = bytes_value.decode('ASCII', 'ignore')
1341
        return self.new_const_cname(value=value)
1342

Stefan Behnel's avatar
Stefan Behnel committed
1343 1344
    def new_num_const_cname(self, value, py_type):
        if py_type == 'long':
1345
            value += 'L'
1346 1347
            py_type = 'int'
        prefix = Naming.interned_prefixes[py_type]
Stefan Behnel's avatar
Stefan Behnel committed
1348
        cname = "%s%s" % (prefix, value)
1349
        cname = cname.replace('+', '_').replace('-', 'neg_').replace('.', '_')
1350 1351
        return cname

1352 1353
    def new_const_cname(self, prefix='', value=''):
        value = replace_identifier('_', value)[:32].strip('_')
1354 1355 1356
        used = self.const_cnames_used
        name_suffix = value
        while name_suffix in used:
1357
            counter = used[value] = used[value] + 1
1358
            name_suffix = '%s_%d' % (value, counter)
1359
        used[name_suffix] = 1
1360 1361 1362 1363 1364
        if prefix:
            prefix = Naming.interned_prefixes[prefix]
        else:
            prefix = Naming.const_prefix
        return "%s%s" % (prefix, name_suffix)
1365

1366 1367
    def get_cached_unbound_method(self, type_cname, method_name):
        key = (type_cname, method_name)
1368 1369 1370 1371 1372 1373 1374
        try:
            cname = self.cached_cmethods[key]
        except KeyError:
            cname = self.cached_cmethods[key] = self.new_const_cname(
                'umethod', '%s_%s' % (type_cname, method_name))
        return cname

1375 1376 1377 1378 1379 1380 1381 1382 1383 1384 1385 1386
    def cached_unbound_method_call_code(self, obj_cname, type_cname, method_name, arg_cnames):
        # admittedly, not the best place to put this method, but it is reused by UtilityCode and ExprNodes ...
        utility_code_name = "CallUnboundCMethod%d" % len(arg_cnames)
        self.use_utility_code(UtilityCode.load_cached(utility_code_name, "ObjectHandling.c"))
        cache_cname = self.get_cached_unbound_method(type_cname, method_name)
        args = [obj_cname] + arg_cnames
        return "__Pyx_%s(&%s, %s)" % (
            utility_code_name,
            cache_cname,
            ', '.join(args),
        )

1387
    def add_cached_builtin_decl(self, entry):
1388
        if entry.is_builtin and entry.is_const:
1389 1390
            if self.should_declare(entry.cname, entry):
                self.put_pyobject_decl(entry)
1391
                w = self.parts['cached_builtins']
1392 1393 1394 1395
                condition = None
                if entry.name in non_portable_builtins_map:
                    condition, replacement = non_portable_builtins_map[entry.name]
                    w.putln('#if %s' % condition)
1396
                    self.put_cached_builtin_init(
1397
                        entry.pos, StringEncoding.EncodedString(replacement),
1398
                        entry.cname)
1399 1400 1401 1402
                    w.putln('#else')
                self.put_cached_builtin_init(
                    entry.pos, StringEncoding.EncodedString(entry.name),
                    entry.cname)
1403
                if condition:
1404 1405 1406 1407 1408
                    w.putln('#endif')

    def put_cached_builtin_init(self, pos, name, cname):
        w = self.parts['cached_builtins']
        interned_cname = self.get_interned_identifier(name).cname
1409 1410 1411
        self.use_utility_code(
            UtilityCode.load_cached("GetBuiltinName", "ObjectHandling.c"))
        w.putln('%s = __Pyx_GetBuiltinName(%s); if (!%s) %s' % (
1412 1413 1414 1415
            cname,
            interned_cname,
            cname,
            w.error_goto(pos)))
1416 1417

    def generate_const_declarations(self):
1418
        self.generate_cached_methods_decls()
1419
        self.generate_string_constants()
Stefan Behnel's avatar
Stefan Behnel committed
1420
        self.generate_num_constants()
1421 1422 1423
        self.generate_object_constant_decls()

    def generate_object_constant_decls(self):
Stefan Behnel's avatar
Stefan Behnel committed
1424 1425
        consts = [(len(c.cname), c.cname, c)
                  for c in self.py_constants]
1426
        consts.sort()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1427
        decls_writer = self.parts['decls']
1428
        for _, cname, c in consts:
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1429
            decls_writer.putln(
1430
                "static %s;" % c.type.declaration_code(cname))
1431

1432 1433 1434 1435
    def generate_cached_methods_decls(self):
        if not self.cached_cmethods:
            return

1436 1437
        decl = self.parts['decls']
        init = self.parts['init_globals']
1438
        cnames = []
1439
        for (type_cname, method_name), cname in sorted(self.cached_cmethods.items()):
1440 1441
            cnames.append(cname)
            method_name_cname = self.get_interned_identifier(StringEncoding.EncodedString(method_name)).cname
1442 1443 1444 1445 1446
            decl.putln('static __Pyx_CachedCFunction %s = {0, &%s, 0, 0, 0};' % (
                cname, method_name_cname))
            # split type reference storage as it might not be static
            init.putln('%s.type = (PyObject*)&%s;' % (
                cname, type_cname))
1447 1448 1449 1450 1451 1452

        if Options.generate_cleanup_code:
            cleanup = self.parts['cleanup_globals']
            for cname in cnames:
                cleanup.putln("Py_CLEAR(%s.method);" % cname)

1453
    def generate_string_constants(self):
1454
        c_consts = [(len(c.cname), c.cname, c) for c in self.string_const_index.values()]
1455 1456
        c_consts.sort()
        py_strings = []
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1457

1458
        decls_writer = self.parts['string_decls']
1459
        for _, cname, c in c_consts:
1460 1461 1462 1463 1464
            conditional = False
            if c.py_versions and (2 not in c.py_versions or 3 not in c.py_versions):
                conditional = True
                decls_writer.putln("#if PY_MAJOR_VERSION %s 3" % (
                    (2 in c.py_versions) and '<' or '>='))
1465
            decls_writer.putln('static const char %s[] = "%s";' % (
1466
                cname, StringEncoding.split_string_literal(c.escaped_value)))
1467 1468
            if conditional:
                decls_writer.putln("#endif")
1469
            if c.py_strings is not None:
Stefan Behnel's avatar
Stefan Behnel committed
1470
                for py_string in c.py_strings.values():
1471 1472
                    py_strings.append((c.cname, len(py_string.cname), py_string))

1473
        for c, cname in sorted(self.pyunicode_ptr_const_index.items()):
1474
            utf16_array, utf32_array = StringEncoding.encode_pyunicode_string(c)
1475 1476
            if utf16_array:
                # Narrow and wide representations differ
Nikita Nemkin's avatar
Nikita Nemkin committed
1477
                decls_writer.putln("#ifdef Py_UNICODE_WIDE")
1478 1479 1480 1481 1482
            decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf32_array))
            if utf16_array:
                decls_writer.putln("#else")
                decls_writer.putln("static Py_UNICODE %s[] = { %s };" % (cname, utf16_array))
                decls_writer.putln("#endif")
1483

1484
        if py_strings:
1485
            self.use_utility_code(UtilityCode.load_cached("InitStrings", "StringTools.c"))
1486
            py_strings.sort()
1487 1488
            w = self.parts['pystring_table']
            w.putln("")
Stefan Behnel's avatar
Stefan Behnel committed
1489
            w.putln("static __Pyx_StringTabEntry %s[] = {" % Naming.stringtab_cname)
1490
            for c_cname, _, py_string in py_strings:
1491
                if not py_string.is_str or not py_string.encoding or \
Stefan Behnel's avatar
Stefan Behnel committed
1492 1493
                        py_string.encoding in ('ASCII', 'USASCII', 'US-ASCII',
                                               'UTF8', 'UTF-8'):
1494 1495 1496 1497
                    encoding = '0'
                else:
                    encoding = '"%s"' % py_string.encoding.lower()

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1498
                decls_writer.putln(
1499
                    "static PyObject *%s;" % py_string.cname)
1500 1501
                if py_string.py3str_cstring:
                    w.putln("#if PY_MAJOR_VERSION >= 3")
Stefan Behnel's avatar
Stefan Behnel committed
1502
                    w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1503 1504 1505
                        py_string.cname,
                        py_string.py3str_cstring.cname,
                        py_string.py3str_cstring.cname,
1506 1507
                        '0', 1, 0,
                        py_string.intern
1508 1509
                        ))
                    w.putln("#else")
Stefan Behnel's avatar
Stefan Behnel committed
1510
                w.putln("{&%s, %s, sizeof(%s), %s, %d, %d, %d}," % (
1511 1512 1513
                    py_string.cname,
                    c_cname,
                    c_cname,
1514 1515 1516 1517
                    encoding,
                    py_string.is_unicode,
                    py_string.is_str,
                    py_string.intern
1518
                    ))
1519 1520
                if py_string.py3str_cstring:
                    w.putln("#endif")
1521
            w.putln("{0, 0, 0, 0, 0, 0, 0}")
1522
            w.putln("};")
1523

1524 1525
            init_globals = self.parts['init_globals']
            init_globals.putln(
1526 1527
                "if (__Pyx_InitStrings(%s) < 0) %s;" % (
                    Naming.stringtab_cname,
1528
                    init_globals.error_goto(self.module_pos)))
1529

Stefan Behnel's avatar
Stefan Behnel committed
1530
    def generate_num_constants(self):
1531
        consts = [(c.py_type, c.value[0] == '-', len(c.value), c.value, c.value_code, c)
Stefan Behnel's avatar
Stefan Behnel committed
1532
                  for c in self.num_const_index.values()]
1533
        consts.sort()
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1534
        decls_writer = self.parts['decls']
Stefan Behnel's avatar
Stefan Behnel committed
1535
        init_globals = self.parts['init_globals']
1536
        for py_type, _, _, value, value_code, c in consts:
1537
            cname = c.cname
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1538
            decls_writer.putln("static PyObject *%s;" % cname)
Stefan Behnel's avatar
Stefan Behnel committed
1539
            if py_type == 'float':
Stefan Behnel's avatar
Stefan Behnel committed
1540
                function = 'PyFloat_FromDouble(%s)'
Stefan Behnel's avatar
Stefan Behnel committed
1541
            elif py_type == 'long':
Stefan Behnel's avatar
Stefan Behnel committed
1542
                function = 'PyLong_FromString((char *)"%s", 0, 0)'
1543
            elif Utils.long_literal(value):
Stefan Behnel's avatar
Stefan Behnel committed
1544
                function = 'PyInt_FromString((char *)"%s", 0, 0)'
1545 1546
            elif len(value.lstrip('-')) > 4:
                function = "PyInt_FromLong(%sL)"
1547
            else:
Stefan Behnel's avatar
Stefan Behnel committed
1548 1549 1550
                function = "PyInt_FromLong(%s)"
            init_globals.putln('%s = %s; %s' % (
                cname, function % value_code,
1551
                init_globals.error_goto_if_null(cname, self.module_pos)))
1552

1553 1554 1555
    # The functions below are there in a transition phase only
    # and will be deprecated. They are called from Nodes.BlockNode.
    # The copy&paste duplication is intentional in order to be able
1556
    # to see quickly how BlockNode worked, until this is replaced.
1557

1558 1559 1560
    def should_declare(self, cname, entry):
        if cname in self.declared_cnames:
            other = self.declared_cnames[cname]
Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
1561
            assert str(entry.type) == str(other.type)
1562 1563 1564 1565 1566 1567 1568 1569 1570
            assert entry.init == other.init
            return False
        else:
            self.declared_cnames[cname] = entry
            return True

    #
    # File name state
    #
1571

1572
    def lookup_filename(self, source_desc):
1573
        entry = source_desc.get_filenametable_entry()
1574
        try:
1575
            index = self.filename_table[entry]
1576 1577
        except KeyError:
            index = len(self.filename_list)
1578
            self.filename_list.append(source_desc)
1579
            self.filename_table[entry] = index
1580 1581 1582 1583 1584 1585
        return index

    def commented_file_contents(self, source_desc):
        try:
            return self.input_file_contents[source_desc]
        except KeyError:
1586 1587 1588 1589
            pass
        source_file = source_desc.get_lines(encoding='ASCII',
                                            error_handling='ignore')
        try:
1590 1591
            F = [u' * ' + line.rstrip().replace(
                    u'*/', u'*[inserted by cython to avoid comment closer]/'
1592 1593
                    ).replace(
                    u'/*', u'/[inserted by cython to avoid comment start]*'
1594
                    )
1595 1596 1597 1598 1599 1600 1601
                 for line in source_file]
        finally:
            if hasattr(source_file, 'close'):
                source_file.close()
        if not F: F.append(u'')
        self.input_file_contents[source_desc] = F
        return F
1602

1603 1604 1605
    #
    # Utility code state
    #
1606

1607
    def use_utility_code(self, utility_code):
1608
        """
1609 1610 1611 1612
        Adds code to the C file. utility_code should
        a) implement __eq__/__hash__ for the purpose of knowing whether the same
           code has already been included
        b) implement put_code, which takes a globalstate instance
1613

1614
        See UtilityCode.
1615
        """
1616
        if utility_code and utility_code not in self.utility_codes:
1617 1618
            self.utility_codes.add(utility_code)
            utility_code.put_code(self)
1619

1620 1621 1622 1623 1624 1625 1626 1627
    def use_entry_utility_code(self, entry):
        if entry is None:
            return
        if entry.utility_code:
            self.use_utility_code(entry.utility_code)
        if entry.utility_code_definition:
            self.use_utility_code(entry.utility_code_definition)

1628

1629 1630
def funccontext_property(func):
    name = func.__name__
1631
    attribute_of = operator.attrgetter(name)
1632
    def get(self):
1633
        return attribute_of(self.funcstate)
1634
    def set(self, value):
1635
        setattr(self.funcstate, name, value)
1636 1637
    return property(get, set)

1638

1639 1640 1641 1642 1643 1644 1645 1646 1647 1648 1649
class CCodeConfig(object):
    # emit_linenums       boolean         write #line pragmas?
    # emit_code_comments  boolean         copy the original code into C comments?
    # c_line_in_traceback boolean         append the c file and line number to the traceback for exceptions?

    def __init__(self, emit_linenums=True, emit_code_comments=True, c_line_in_traceback=True):
        self.emit_code_comments = emit_code_comments
        self.emit_linenums = emit_linenums
        self.c_line_in_traceback = c_line_in_traceback


1650
class CCodeWriter(object):
1651
    """
1652
    Utility class to output C code.
1653

1654
    When creating an insertion point one must care about the state that is
1655
    kept:
1656 1657
    - formatting state (level, bol) is cloned and used in insertion points
      as well
1658 1659
    - labels, temps, exc_vars: One must construct a scope in which these can
      exist by calling enter_cfunc_scope/exit_cfunc_scope (these are for
luz.paz's avatar
luz.paz committed
1660
      sanity checking and forward compatibility). Created insertion points
1661 1662 1663 1664
      looses this scope and cannot access it.
    - marker: Not copied to insertion point
    - filename_table, filename_list, input_file_contents: All codewriters
      coming from the same root share the same instances simultaneously.
1665
    """
1666

1667 1668 1669 1670 1671 1672 1673 1674 1675 1676 1677 1678
    # f                   file            output file
    # buffer              StringIOTree

    # level               int             indentation level
    # bol                 bool            beginning of line?
    # marker              string          comment to emit before next line
    # funcstate           FunctionState   contains state local to a C function used for code
    #                                     generation (labels and temps state etc.)
    # globalstate         GlobalState     contains state global for a C file (input file info,
    #                                     utility code, declared constants etc.)
    # pyclass_stack       list            used during recursive code generation to pass information
    #                                     about the current class one is in
1679
    # code_config         CCodeConfig     configuration options for the C code writer
1680

1681
    @cython.locals(create_from='CCodeWriter')
1682
    def __init__(self, create_from=None, buffer=None, copy_formatting=False):
1683 1684
        if buffer is None: buffer = StringIOTree()
        self.buffer = buffer
1685 1686
        self.last_pos = None
        self.last_marked_pos = None
1687
        self.pyclass_stack = []
1688

1689
        self.funcstate = None
1690 1691
        self.globalstate = None
        self.code_config = None
1692
        self.level = 0
Robert Bradshaw's avatar
Robert Bradshaw committed
1693
        self.call_level = 0
1694
        self.bol = 1
1695 1696

        if create_from is not None:
1697
            # Use same global state
1698
            self.set_global_state(create_from.globalstate)
1699
            self.funcstate = create_from.funcstate
1700
            # Clone formatting state
1701 1702 1703
            if copy_formatting:
                self.level = create_from.level
                self.bol = create_from.bol
Robert Bradshaw's avatar
Robert Bradshaw committed
1704
                self.call_level = create_from.call_level
1705 1706
            self.last_pos = create_from.last_pos
            self.last_marked_pos = create_from.last_marked_pos
1707

1708
    def create_new(self, create_from, buffer, copy_formatting):
1709 1710
        # polymorphic constructor -- very slightly more versatile
        # than using __class__
1711
        result = CCodeWriter(create_from, buffer, copy_formatting)
1712
        return result
1713

1714 1715 1716 1717 1718
    def set_global_state(self, global_state):
        assert self.globalstate is None  # prevent overwriting once it's set
        self.globalstate = global_state
        self.code_config = global_state.code_config

1719 1720 1721
    def copyto(self, f):
        self.buffer.copyto(f)

1722 1723 1724 1725
    def getvalue(self):
        return self.buffer.getvalue()

    def write(self, s):
1726
        # also put invalid markers (lineno 0), to indicate that those lines
Mark Florisson's avatar
Mark Florisson committed
1727
        # have no Cython source code correspondence
1728
        cython_lineno = self.last_marked_pos[1] if self.last_marked_pos else 0
Mark Florisson's avatar
Mark Florisson committed
1729
        self.buffer.markers.extend([cython_lineno] * s.count('\n'))
1730
        self.buffer.write(s)
1731 1732

    def insertion_point(self):
1733
        other = self.create_new(create_from=self, buffer=self.buffer.insertion_point(), copy_formatting=True)
1734 1735
        return other

1736 1737 1738 1739 1740
    def new_writer(self):
        """
        Creates a new CCodeWriter connected to the same global state, which
        can later be inserted using insert.
        """
1741
        return CCodeWriter(create_from=self)
1742 1743 1744 1745 1746 1747 1748 1749 1750 1751

    def insert(self, writer):
        """
        Inserts the contents of another code writer (created with
        the same global state) in the current location.

        It is ok to write to the inserted writer also after insertion.
        """
        assert writer.globalstate is self.globalstate
        self.buffer.insert(writer.buffer)
1752 1753

    # Properties delegated to function scope
1754 1755 1756 1757 1758 1759 1760 1761 1762 1763 1764 1765 1766 1767 1768 1769
    @funccontext_property
    def label_counter(self): pass
    @funccontext_property
    def return_label(self): pass
    @funccontext_property
    def error_label(self): pass
    @funccontext_property
    def labels_used(self): pass
    @funccontext_property
    def continue_label(self): pass
    @funccontext_property
    def break_label(self): pass
    @funccontext_property
    def return_from_error_cleanup_label(self): pass
    @funccontext_property
    def yield_labels(self): pass
1770 1771

    # Functions delegated to function scope
Dag Sverre Seljebotn's avatar
merge  
Dag Sverre Seljebotn committed
1772
    def new_label(self, name=None):    return self.funcstate.new_label(name)
1773
    def new_error_label(self):         return self.funcstate.new_error_label()
1774
    def new_yield_label(self, *args):  return self.funcstate.new_yield_label(*args)
1775 1776 1777 1778 1779 1780 1781 1782
    def get_loop_labels(self):         return self.funcstate.get_loop_labels()
    def set_loop_labels(self, labels): return self.funcstate.set_loop_labels(labels)
    def new_loop_labels(self):         return self.funcstate.new_loop_labels()
    def get_all_labels(self):          return self.funcstate.get_all_labels()
    def set_all_labels(self, labels):  return self.funcstate.set_all_labels(labels)
    def all_new_labels(self):          return self.funcstate.all_new_labels()
    def use_label(self, lbl):          return self.funcstate.use_label(lbl)
    def label_used(self, lbl):         return self.funcstate.label_used(lbl)
1783 1784


1785 1786
    def enter_cfunc_scope(self, scope=None):
        self.funcstate = FunctionState(self, scope=scope)
1787

1788
    def exit_cfunc_scope(self):
1789
        self.funcstate = None
1790

1791 1792
    # constant handling

Stefan Behnel's avatar
Stefan Behnel committed
1793
    def get_py_int(self, str_value, longness):
1794 1795
        return self.globalstate.get_int_const(str_value, longness).cname

Stefan Behnel's avatar
Stefan Behnel committed
1796 1797 1798
    def get_py_float(self, str_value, value_code):
        return self.globalstate.get_float_const(str_value, value_code).cname

1799 1800
    def get_py_const(self, type, prefix='', cleanup_level=None, dedup_key=None):
        return self.globalstate.get_py_const(type, prefix, cleanup_level, dedup_key).cname
1801

1802 1803 1804
    def get_string_const(self, text):
        return self.globalstate.get_string_const(text).cname

1805 1806
    def get_pyunicode_ptr_const(self, text):
        return self.globalstate.get_pyunicode_ptr_const(text)
1807

1808 1809 1810 1811
    def get_py_string_const(self, text, identifier=None,
                            is_str=False, unicode_value=None):
        return self.globalstate.get_py_string_const(
            text, identifier, is_str, unicode_value).cname
1812

1813 1814 1815
    def get_argument_default_const(self, type):
        return self.globalstate.get_py_const(type).cname

1816 1817 1818 1819
    def intern(self, text):
        return self.get_py_string_const(text)

    def intern_identifier(self, text):
1820
        return self.get_py_string_const(text, identifier=True)
1821

1822 1823
    def get_cached_constants_writer(self, target=None):
        return self.globalstate.get_cached_constants_writer(target)
1824

1825 1826
    # code generation

Stefan Behnel's avatar
Stefan Behnel committed
1827
    def putln(self, code="", safe=False):
1828
        if self.last_pos and self.bol:
William Stein's avatar
William Stein committed
1829
            self.emit_marker()
1830
        if self.code_config.emit_linenums and self.last_marked_pos:
1831 1832
            source_desc, line, _ = self.last_marked_pos
            self.write('\n#line %s "%s"\n' % (line, source_desc.get_escaped_description()))
William Stein's avatar
William Stein committed
1833
        if code:
1834 1835 1836 1837
            if safe:
                self.put_safe(code)
            else:
                self.put(code)
1838
        self.write("\n")
William Stein's avatar
William Stein committed
1839
        self.bol = 1
1840

1841
    def mark_pos(self, pos, trace=True):
1842 1843 1844 1845
        if pos is None:
            return
        if self.last_marked_pos and self.last_marked_pos[:2] == pos[:2]:
            return
1846
        self.last_pos = (pos, trace)
1847

William Stein's avatar
William Stein committed
1848
    def emit_marker(self):
1849 1850
        pos, trace = self.last_pos
        self.last_marked_pos = pos
1851
        self.last_pos = None
1852
        self.write("\n")
1853
        if self.code_config.emit_code_comments:
1854 1855
            self.indent()
            self.write("/* %s */\n" % self._build_marker(pos))
1856
        if trace and self.funcstate and self.funcstate.can_trace and self.globalstate.directives['linetrace']:
1857
            self.indent()
1858 1859 1860 1861 1862 1863 1864 1865 1866 1867 1868
            self.write('__Pyx_TraceLine(%d,%d,%s)\n' % (
                pos[1], not self.funcstate.gil_owned, self.error_goto(pos)))

    def _build_marker(self, pos):
        source_desc, line, col = pos
        assert isinstance(source_desc, SourceDescriptor)
        contents = self.globalstate.commented_file_contents(source_desc)
        lines = contents[max(0, line-3):line]  # line numbers start at 1
        lines[-1] += u'             # <<<<<<<<<<<<<<'
        lines += contents[line:line+2]
        return u'"%s":%d\n%s\n' % (source_desc.get_escaped_description(), line, u'\n'.join(lines))
William Stein's avatar
William Stein committed
1869

1870 1871 1872 1873 1874
    def put_safe(self, code):
        # put code, but ignore {}
        self.write(code)
        self.bol = 0

1875
    def put_or_include(self, code, name):
Stefan Behnel's avatar
Stefan Behnel committed
1876
        include_dir = self.globalstate.common_utility_include_dir
1877
        if include_dir and len(code) > 1024:
Stefan Behnel's avatar
Stefan Behnel committed
1878
            include_file = "%s_%s.h" % (
1879
                name, hashlib.sha1(code.encode('utf8')).hexdigest())
Stefan Behnel's avatar
Stefan Behnel committed
1880 1881 1882
            path = os.path.join(include_dir, include_file)
            if not os.path.exists(path):
                tmp_path = '%s.tmp%s' % (path, os.getpid())
Stefan Behnel's avatar
Stefan Behnel committed
1883
                with closing(Utils.open_new_file(tmp_path)) as f:
Stefan Behnel's avatar
Stefan Behnel committed
1884
                    f.write(code)
1885
                shutil.move(tmp_path, path)
Stefan Behnel's avatar
Stefan Behnel committed
1886 1887
            code = '#include "%s"\n' % path
        self.put(code)
1888

William Stein's avatar
William Stein committed
1889
    def put(self, code):
1890
        fix_indent = False
1891 1892 1893 1894 1895 1896 1897 1898 1899 1900 1901 1902
        if "{" in code:
            dl = code.count("{")
        else:
            dl = 0
        if "}" in code:
            dl -= code.count("}")
            if dl < 0:
                self.level += dl
            elif dl == 0 and code[0] == "}":
                # special cases like "} else {" need a temporary dedent
                fix_indent = True
                self.level -= 1
William Stein's avatar
William Stein committed
1903 1904
        if self.bol:
            self.indent()
1905
        self.write(code)
William Stein's avatar
William Stein committed
1906 1907 1908
        self.bol = 0
        if dl > 0:
            self.level += dl
1909
        elif fix_indent:
1910 1911
            self.level += 1

Mark Florisson's avatar
Mark Florisson committed
1912
    def putln_tempita(self, code, **context):
1913
        from ..Tempita import sub
1914
        self.putln(sub(code, **context))
Mark Florisson's avatar
Mark Florisson committed
1915 1916

    def put_tempita(self, code, **context):
1917
        from ..Tempita import sub
1918
        self.put(sub(code, **context))
Mark Florisson's avatar
Mark Florisson committed
1919

William Stein's avatar
William Stein committed
1920
    def increase_indent(self):
Stefan Behnel's avatar
Stefan Behnel committed
1921
        self.level += 1
1922

William Stein's avatar
William Stein committed
1923
    def decrease_indent(self):
Stefan Behnel's avatar
Stefan Behnel committed
1924
        self.level -= 1
1925

William Stein's avatar
William Stein committed
1926 1927 1928
    def begin_block(self):
        self.putln("{")
        self.increase_indent()
1929

William Stein's avatar
William Stein committed
1930 1931 1932
    def end_block(self):
        self.decrease_indent()
        self.putln("}")
1933

William Stein's avatar
William Stein committed
1934
    def indent(self):
1935
        self.write("  " * self.level)
1936

1937 1938 1939
    def get_py_version_hex(self, pyversion):
        return "0x%02X%02X%02X%02X" % (tuple(pyversion) + (0,0,0,0))[:4]

William Stein's avatar
William Stein committed
1940
    def put_label(self, lbl):
1941
        if lbl in self.funcstate.labels_used:
1942
            self.putln("%s:;" % lbl)
1943

1944
    def put_goto(self, lbl):
1945
        self.funcstate.use_label(lbl)
1946
        self.putln("goto %s;" % lbl)
1947

1948
    def put_var_declaration(self, entry, storage_class="",
Stefan Behnel's avatar
Stefan Behnel committed
1949
                            dll_linkage=None, definition=True):
1950
        #print "Code.put_var_declaration:", entry.name, "definition =", definition ###
1951 1952
        if entry.visibility == 'private' and not (definition or entry.defined_in_pxd):
            #print "...private and not definition, skipping", entry.cname ###
1953
            return
1954 1955
        if entry.visibility == "private" and not entry.used:
            #print "...private and not used, skipping", entry.cname ###
William Stein's avatar
William Stein committed
1956 1957 1958
            return
        if storage_class:
            self.put("%s " % storage_class)
1959 1960
        if not entry.cf_used:
            self.put('CYTHON_UNUSED ')
1961
        self.put(entry.type.declaration_code(
Stefan Behnel's avatar
Stefan Behnel committed
1962
            entry.cname, dll_linkage=dll_linkage))
William Stein's avatar
William Stein committed
1963
        if entry.init is not None:
1964
            self.put_safe(" = %s" % entry.type.literal_code(entry.init))
1965
        elif entry.type.is_pyobject:
Stefan Behnel's avatar
Stefan Behnel committed
1966
            self.put(" = NULL")
William Stein's avatar
William Stein committed
1967
        self.putln(";")
1968 1969

    def put_temp_declarations(self, func_context):
1970
        for name, type, manage_ref, static in func_context.temps_allocated:
1971 1972 1973
            decl = type.declaration_code(name)
            if type.is_pyobject:
                self.putln("%s = NULL;" % decl)
1974
            elif type.is_memoryviewslice:
1975
                from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
1976
                self.putln("%s = %s;" % (decl, MemoryView.memslice_entry_init))
1977
            else:
1978
                self.putln("%s%s;" % (static and "static " or "", decl))
1979

1980
        if func_context.should_declare_error_indicator:
1981 1982 1983 1984
            if self.funcstate.uses_error_indicator:
                unused = ''
            else:
                unused = 'CYTHON_UNUSED '
1985
            # Initialize these variables to silence compiler warnings
1986 1987 1988
            self.putln("%sint %s = 0;" % (unused, Naming.lineno_cname))
            self.putln("%sconst char *%s = NULL;" % (unused, Naming.filename_cname))
            self.putln("%sint %s = 0;" % (unused, Naming.clineno_cname))
1989

1990 1991 1992 1993
    def put_generated_by(self):
        self.putln("/* Generated by Cython %s */" % Version.watermark)
        self.putln("")

1994 1995 1996
    def put_h_guard(self, guard):
        self.putln("#ifndef %s" % guard)
        self.putln("#define %s" % guard)
1997

1998 1999 2000 2001 2002 2003
    def unlikely(self, cond):
        if Options.gcc_branch_hints:
            return 'unlikely(%s)' % cond
        else:
            return cond

2004 2005 2006 2007 2008
    def build_function_modifiers(self, modifiers, mapper=modifier_output_mapper):
        if not modifiers:
            return ''
        return '%s ' % ' '.join([mapper(m,m) for m in modifiers])

2009 2010
    # Python objects and reference counting

William Stein's avatar
William Stein committed
2011 2012
    def entry_as_pyobject(self, entry):
        type = entry.type
2013 2014
        if (not entry.is_self_arg and not entry.type.is_complete()
            or entry.type.is_extension_type):
William Stein's avatar
William Stein committed
2015 2016 2017
            return "(PyObject *)" + entry.cname
        else:
            return entry.cname
2018

William Stein's avatar
William Stein committed
2019
    def as_pyobject(self, cname, type):
2020
        from .PyrexTypes import py_object_type, typecast
William Stein's avatar
William Stein committed
2021
        return typecast(py_object_type, type, cname)
2022

2023
    def put_gotref(self, cname):
2024
        self.putln("__Pyx_GOTREF(%s);" % cname)
2025

2026 2027
    def put_giveref(self, cname):
        self.putln("__Pyx_GIVEREF(%s);" % cname)
2028

2029 2030 2031
    def put_xgiveref(self, cname):
        self.putln("__Pyx_XGIVEREF(%s);" % cname)

Dag Sverre Seljebotn's avatar
Dag Sverre Seljebotn committed
2032 2033 2034
    def put_xgotref(self, cname):
        self.putln("__Pyx_XGOTREF(%s);" % cname)

2035 2036 2037 2038 2039
    def put_incref(self, cname, type, nanny=True):
        if nanny:
            self.putln("__Pyx_INCREF(%s);" % self.as_pyobject(cname, type))
        else:
            self.putln("Py_INCREF(%s);" % self.as_pyobject(cname, type))
2040

2041
    def put_decref(self, cname, type, nanny=True):
2042
        self._put_decref(cname, type, nanny, null_check=False, clear=False)
2043 2044 2045 2046

    def put_var_gotref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_GOTREF(%s);" % self.entry_as_pyobject(entry))
2047

2048 2049 2050 2051
    def put_var_giveref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_GIVEREF(%s);" % self.entry_as_pyobject(entry))

2052 2053 2054 2055
    def put_var_xgotref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XGOTREF(%s);" % self.entry_as_pyobject(entry))

2056 2057 2058 2059
    def put_var_xgiveref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XGIVEREF(%s);" % self.entry_as_pyobject(entry))

2060
    def put_var_incref(self, entry, nanny=True):
William Stein's avatar
William Stein committed
2061
        if entry.type.is_pyobject:
2062 2063 2064 2065
            if nanny:
                self.putln("__Pyx_INCREF(%s);" % self.entry_as_pyobject(entry))
            else:
                self.putln("Py_INCREF(%s);" % self.entry_as_pyobject(entry))
2066

2067 2068 2069 2070
    def put_var_xincref(self, entry):
        if entry.type.is_pyobject:
            self.putln("__Pyx_XINCREF(%s);" % self.entry_as_pyobject(entry))

2071 2072 2073
    def put_decref_clear(self, cname, type, nanny=True, clear_before_decref=False):
        self._put_decref(cname, type, nanny, null_check=False,
                         clear=True, clear_before_decref=clear_before_decref)
2074

2075
    def put_xdecref(self, cname, type, nanny=True, have_gil=True):
2076 2077
        self._put_decref(cname, type, nanny, null_check=True,
                         have_gil=have_gil, clear=False)
2078

2079 2080 2081
    def put_xdecref_clear(self, cname, type, nanny=True, clear_before_decref=False):
        self._put_decref(cname, type, nanny, null_check=True,
                         clear=True, clear_before_decref=clear_before_decref)
2082

2083 2084
    def _put_decref(self, cname, type, nanny=True, null_check=False,
                    have_gil=True, clear=False, clear_before_decref=False):
2085
        if type.is_memoryviewslice:
2086
            self.put_xdecref_memoryviewslice(cname, have_gil=have_gil)
2087 2088
            return

2089 2090
        prefix = '__Pyx' if nanny else 'Py'
        X = 'X' if null_check else ''
2091 2092 2093 2094 2095 2096 2097 2098 2099

        if clear:
            if clear_before_decref:
                if not nanny:
                    X = ''  # CPython doesn't have a Py_XCLEAR()
                self.putln("%s_%sCLEAR(%s);" % (prefix, X, cname))
            else:
                self.putln("%s_%sDECREF(%s); %s = 0;" % (
                    prefix, X, self.as_pyobject(cname, type), cname))
2100
        else:
2101 2102
            self.putln("%s_%sDECREF(%s);" % (
                prefix, X, self.as_pyobject(cname, type)))
William Stein's avatar
William Stein committed
2103

2104 2105 2106 2107 2108 2109
    def put_decref_set(self, cname, rhs_cname):
        self.putln("__Pyx_DECREF_SET(%s, %s);" % (cname, rhs_cname))

    def put_xdecref_set(self, cname, rhs_cname):
        self.putln("__Pyx_XDECREF_SET(%s, %s);" % (cname, rhs_cname))

William Stein's avatar
William Stein committed
2110 2111
    def put_var_decref(self, entry):
        if entry.type.is_pyobject:
2112
            self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
2113

2114
    def put_var_xdecref(self, entry, nanny=True):
William Stein's avatar
William Stein committed
2115
        if entry.type.is_pyobject:
2116 2117 2118 2119
            if nanny:
                self.putln("__Pyx_XDECREF(%s);" % self.entry_as_pyobject(entry))
            else:
                self.putln("Py_XDECREF(%s);" % self.entry_as_pyobject(entry))
2120

2121 2122 2123
    def put_var_decref_clear(self, entry):
        self._put_var_decref_clear(entry, null_check=False)

William Stein's avatar
William Stein committed
2124
    def put_var_xdecref_clear(self, entry):
2125 2126 2127
        self._put_var_decref_clear(entry, null_check=True)

    def _put_var_decref_clear(self, entry, null_check):
William Stein's avatar
William Stein committed
2128
        if entry.type.is_pyobject:
2129 2130 2131 2132 2133 2134 2135 2136 2137
            if entry.in_closure:
                # reset before DECREF to make sure closure state is
                # consistent during call to DECREF()
                self.putln("__Pyx_%sCLEAR(%s);" % (
                    null_check and 'X' or '',
                    entry.cname))
            else:
                self.putln("__Pyx_%sDECREF(%s); %s = 0;" % (
                    null_check and 'X' or '',
2138
                    self.entry_as_pyobject(entry),
2139
                    entry.cname))
2140

2141
    def put_var_decrefs(self, entries, used_only = 0):
William Stein's avatar
William Stein committed
2142
        for entry in entries:
2143 2144 2145 2146 2147
            if not used_only or entry.used:
                if entry.xdecref_cleanup:
                    self.put_var_xdecref(entry)
                else:
                    self.put_var_decref(entry)
2148

William Stein's avatar
William Stein committed
2149 2150 2151
    def put_var_xdecrefs(self, entries):
        for entry in entries:
            self.put_var_xdecref(entry)
2152

William Stein's avatar
William Stein committed
2153 2154 2155
    def put_var_xdecrefs_clear(self, entries):
        for entry in entries:
            self.put_var_xdecref_clear(entry)
2156

2157
    def put_incref_memoryviewslice(self, slice_cname, have_gil=False):
2158
        from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
2159
        self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
2160 2161 2162
        self.putln("__PYX_INC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    def put_xdecref_memoryviewslice(self, slice_cname, have_gil=False):
2163
        from . import MemoryView
Mark Florisson's avatar
Mark Florisson committed
2164
        self.globalstate.use_utility_code(MemoryView.memviewslice_init_code)
2165 2166 2167 2168 2169
        self.putln("__PYX_XDEC_MEMVIEW(&%s, %d);" % (slice_cname, int(have_gil)))

    def put_xgiveref_memoryviewslice(self, slice_cname):
        self.put_xgiveref("%s.memview" % slice_cname)

2170
    def put_init_to_py_none(self, cname, type, nanny=True):
2171
        from .PyrexTypes import py_object_type, typecast
William Stein's avatar
William Stein committed
2172
        py_none = typecast(type, py_object_type, "Py_None")
2173 2174 2175 2176
        if nanny:
            self.putln("%s = %s; __Pyx_INCREF(Py_None);" % (cname, py_none))
        else:
            self.putln("%s = %s; Py_INCREF(Py_None);" % (cname, py_none))
2177

2178
    def put_init_var_to_py_none(self, entry, template = "%s", nanny=True):
William Stein's avatar
William Stein committed
2179 2180
        code = template % entry.cname
        #if entry.type.is_extension_type:
Robert Bradshaw's avatar
Robert Bradshaw committed
2181
        #    code = "((PyObject*)%s)" % code
2182
        self.put_init_to_py_none(code, entry.type, nanny)
2183
        if entry.in_closure:
Stefan Behnel's avatar
Stefan Behnel committed
2184
            self.put_giveref('Py_None')
William Stein's avatar
William Stein committed
2185

2186
    def put_pymethoddef(self, entry, term, allow_skip=True, wrapper_code_writer=None):
2187
        if entry.is_special or entry.name == '__getattribute__':
2188
            if entry.name not in special_py_methods:
2189 2190
                if entry.name == '__getattr__' and not self.globalstate.directives['fast_getattr']:
                    pass
2191 2192 2193
                # Python's typeobject.c will automatically fill in our slot
                # in add_operators() (called by PyType_Ready) with a value
                # that's better than ours.
2194
                elif allow_skip:
2195
                    return
2196

2197
        method_flags = entry.signature.method_flags()
2198 2199 2200 2201 2202 2203 2204
        if not method_flags:
            return
        if entry.is_special:
            from . import TypeSlots
            method_flags += [TypeSlots.method_coexist]
        func_ptr = wrapper_code_writer.put_pymethoddef_wrapper(entry) if wrapper_code_writer else entry.func_cname
        # Add required casts, but try not to shadow real warnings.
2205
        cast = entry.signature.method_function_type()
2206 2207 2208 2209 2210 2211 2212 2213 2214 2215 2216 2217 2218
        if cast != 'PyCFunction':
            func_ptr = '(void*)(%s)%s' % (cast, func_ptr)
        self.putln(
            '{"%s", (PyCFunction)%s, %s, %s}%s' % (
                entry.name,
                func_ptr,
                "|".join(method_flags),
                entry.doc_cname if entry.doc else '0',
                term))

    def put_pymethoddef_wrapper(self, entry):
        func_cname = entry.func_cname
        if entry.is_special:
2219 2220 2221
            method_flags = entry.signature.method_flags() or []
            from .TypeSlots import method_noargs
            if method_noargs in method_flags:
2222 2223 2224 2225 2226
                # Special NOARGS methods really take no arguments besides 'self', but PyCFunction expects one.
                func_cname = Naming.method_wrapper_prefix + func_cname
                self.putln("static PyObject *%s(PyObject *self, CYTHON_UNUSED PyObject *arg) {return %s(self);}" % (
                    func_cname, entry.func_cname))
        return func_cname
2227

2228 2229
    # GIL methods

2230 2231 2232 2233 2234 2235
    def use_fast_gil_utility_code(self):
        if self.globalstate.directives['fast_gil']:
            self.globalstate.use_utility_code(UtilityCode.load_cached("FastGil", "ModuleSetupCode.c"))
        else:
            self.globalstate.use_utility_code(UtilityCode.load_cached("NoFastGil", "ModuleSetupCode.c"))

2236
    def put_ensure_gil(self, declare_gilstate=True, variable=None):
2237 2238 2239 2240 2241 2242
        """
        Acquire the GIL. The generated code is safe even when no PyThreadState
        has been allocated for this thread (for threads not initialized by
        using the Python API). Additionally, the code generated by this method
        may be called recursively.
        """
2243 2244
        self.globalstate.use_utility_code(
            UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
2245
        self.use_fast_gil_utility_code()
2246
        self.putln("#ifdef WITH_THREAD")
2247 2248 2249 2250
        if not variable:
            variable = '__pyx_gilstate_save'
            if declare_gilstate:
                self.put("PyGILState_STATE ")
2251
        self.putln("%s = __Pyx_PyGILState_Ensure();" % variable)
2252 2253
        self.putln("#endif")

2254
    def put_release_ensured_gil(self, variable=None):
2255 2256 2257
        """
        Releases the GIL, corresponds to `put_ensure_gil`.
        """
2258
        self.use_fast_gil_utility_code()
2259 2260
        if not variable:
            variable = '__pyx_gilstate_save'
2261
        self.putln("#ifdef WITH_THREAD")
2262
        self.putln("__Pyx_PyGILState_Release(%s);" % variable)
2263 2264
        self.putln("#endif")

2265
    def put_acquire_gil(self, variable=None):
2266 2267 2268 2269
        """
        Acquire the GIL. The thread's thread state must have been initialized
        by a previous `put_release_gil`
        """
2270
        self.use_fast_gil_utility_code()
2271
        self.putln("#ifdef WITH_THREAD")
2272
        self.putln("__Pyx_FastGIL_Forget();")
2273 2274
        if variable:
            self.putln('_save = %s;' % variable)
2275
        self.putln("Py_BLOCK_THREADS")
2276
        self.putln("#endif")
2277

2278
    def put_release_gil(self, variable=None):
2279
        "Release the GIL, corresponds to `put_acquire_gil`."
2280
        self.use_fast_gil_utility_code()
2281
        self.putln("#ifdef WITH_THREAD")
2282
        self.putln("PyThreadState *_save;")
2283
        self.putln("Py_UNBLOCK_THREADS")
2284 2285
        if variable:
            self.putln('%s = _save;' % variable)
2286
        self.putln("__Pyx_FastGIL_Remember();")
2287
        self.putln("#endif")
2288

2289 2290
    def declare_gilstate(self):
        self.putln("#ifdef WITH_THREAD")
2291
        self.putln("PyGILState_STATE __pyx_gilstate_save;")
2292 2293
        self.putln("#endif")

2294 2295
    # error handling

Robert Bradshaw's avatar
Robert Bradshaw committed
2296
    def put_error_if_neg(self, pos, value):
2297 2298
        # TODO this path is almost _never_ taken, yet this macro makes is slower!
        # return self.putln("if (unlikely(%s < 0)) %s" % (value, self.error_goto(pos)))
Robert Bradshaw's avatar
Robert Bradshaw committed
2299 2300
        return self.putln("if (%s < 0) %s" % (value, self.error_goto(pos)))

2301
    def put_error_if_unbound(self, pos, entry, in_nogil_context=False):
2302
        from . import ExprNodes
2303
        if entry.from_closure:
2304 2305 2306
            func = '__Pyx_RaiseClosureNameError'
            self.globalstate.use_utility_code(
                ExprNodes.raise_closure_name_error_utility_code)
2307 2308 2309 2310
        elif entry.type.is_memoryviewslice and in_nogil_context:
            func = '__Pyx_RaiseUnboundMemoryviewSliceNogil'
            self.globalstate.use_utility_code(
                ExprNodes.raise_unbound_memoryview_utility_code_nogil)
2311
        else:
2312 2313 2314
            func = '__Pyx_RaiseUnboundLocalError'
            self.globalstate.use_utility_code(
                ExprNodes.raise_unbound_local_error_utility_code)
Mark Florisson's avatar
Mark Florisson committed
2315

2316
        self.putln('if (unlikely(!%s)) { %s("%s"); %s }' % (
Mark Florisson's avatar
Mark Florisson committed
2317 2318 2319 2320
                                entry.type.check_for_null_code(entry.cname),
                                func,
                                entry.name,
                                self.error_goto(pos)))
2321

2322
    def set_error_info(self, pos, used=False):
2323
        self.funcstate.should_declare_error_indicator = True
2324 2325
        if used:
            self.funcstate.uses_error_indicator = True
2326
        if self.code_config.c_line_in_traceback:
2327
            cinfo = " %s = %s;" % (Naming.clineno_cname, Naming.line_c_macro)
Robert Bradshaw's avatar
Robert Bradshaw committed
2328 2329
        else:
            cinfo = ""
2330

2331
        return "%s = %s[%s]; %s = %s;%s" % (
William Stein's avatar
William Stein committed
2332 2333 2334 2335 2336
            Naming.filename_cname,
            Naming.filetable_cname,
            self.lookup_filename(pos[0]),
            Naming.lineno_cname,
            pos[1],
2337
            cinfo)
2338

2339 2340 2341
    def error_goto(self, pos):
        lbl = self.funcstate.error_label
        self.funcstate.use_label(lbl)
2342 2343
        if not pos:
            return 'goto %s;' % lbl
2344 2345 2346
        return "__PYX_ERR(%s, %s, %s)" % (
            self.lookup_filename(pos[0]),
            pos[1],
2347
            lbl)
2348 2349 2350

    def error_goto_if(self, cond, pos):
        return "if (%s) %s" % (self.unlikely(cond), self.error_goto(pos))
2351

Robert Bradshaw's avatar
Robert Bradshaw committed
2352 2353
    def error_goto_if_null(self, cname, pos):
        return self.error_goto_if("!%s" % cname, pos)
2354

Robert Bradshaw's avatar
Robert Bradshaw committed
2355 2356
    def error_goto_if_neg(self, cname, pos):
        return self.error_goto_if("%s < 0" % cname, pos)
2357

Robert Bradshaw's avatar
Robert Bradshaw committed
2358 2359
    def error_goto_if_PyErr(self, pos):
        return self.error_goto_if("PyErr_Occurred()", pos)
2360

William Stein's avatar
William Stein committed
2361
    def lookup_filename(self, filename):
2362
        return self.globalstate.lookup_filename(filename)
William Stein's avatar
William Stein committed
2363

2364
    def put_declare_refcount_context(self):
2365
        self.putln('__Pyx_RefNannyDeclarations')
2366

2367 2368
    def put_setup_refcount_context(self, name, acquire_gil=False):
        if acquire_gil:
2369 2370
            self.globalstate.use_utility_code(
                UtilityCode.load_cached("ForceInitThreads", "ModuleSetupCode.c"))
2371
        self.putln('__Pyx_RefNannySetupContext("%s", %d);' % (name, acquire_gil and 1 or 0))
2372

2373
    def put_finish_refcount_context(self):
2374
        self.putln("__Pyx_RefNannyFinishContext();")
2375

2376
    def put_add_traceback(self, qualified_name, include_cline=True):
2377 2378 2379
        """
        Build a Python traceback for propagating exceptions.

2380
        qualified_name should be the qualified name of the function.
2381 2382 2383
        """
        format_tuple = (
            qualified_name,
2384
            Naming.clineno_cname if include_cline else 0,
2385 2386 2387
            Naming.lineno_cname,
            Naming.filename_cname,
        )
2388
        self.funcstate.uses_error_indicator = True
2389 2390
        self.putln('__Pyx_AddTraceback("%s", %s, %s, %s);' % format_tuple)

2391
    def put_unraisable(self, qualified_name, nogil=False):
2392 2393 2394 2395 2396 2397 2398 2399 2400 2401
        """
        Generate code to print a Python warning for an unraisable exception.

        qualified_name should be the qualified name of the function.
        """
        format_tuple = (
            qualified_name,
            Naming.clineno_cname,
            Naming.lineno_cname,
            Naming.filename_cname,
2402 2403
            self.globalstate.directives['unraisable_tracebacks'],
            nogil,
2404 2405
        )
        self.funcstate.uses_error_indicator = True
2406
        self.putln('__Pyx_WriteUnraisable("%s", %s, %s, %s, %d, %d);' % format_tuple)
2407 2408 2409
        self.globalstate.use_utility_code(
            UtilityCode.load_cached("WriteUnraisableException", "Exceptions.c"))

2410 2411 2412 2413
    def put_trace_declarations(self):
        self.putln('__Pyx_TraceDeclarations')

    def put_trace_frame_init(self, codeobj=None):
2414 2415
        if codeobj:
            self.putln('__Pyx_TraceFrameInit(%s)' % codeobj)
2416

2417
    def put_trace_call(self, name, pos, nogil=False):
2418 2419
        self.putln('__Pyx_TraceCall("%s", %s[%s], %s, %d, %s);' % (
            name, Naming.filetable_cname, self.lookup_filename(pos[0]), pos[1], nogil, self.error_goto(pos)))
2420

Robert Bradshaw's avatar
Robert Bradshaw committed
2421 2422
    def put_trace_exception(self):
        self.putln("__Pyx_TraceException();")
2423

2424 2425
    def put_trace_return(self, retvalue_cname, nogil=False):
        self.putln("__Pyx_TraceReturn(%s, %d);" % (retvalue_cname, nogil))
Robert Bradshaw's avatar
Robert Bradshaw committed
2426

Mark Florisson's avatar
Mark Florisson committed
2427 2428 2429
    def putln_openmp(self, string):
        self.putln("#ifdef _OPENMP")
        self.putln(string)
2430
        self.putln("#endif /* _OPENMP */")
William Stein's avatar
William Stein committed
2431

2432 2433 2434 2435 2436 2437 2438 2439 2440 2441 2442 2443 2444 2445 2446 2447 2448 2449 2450 2451
    def undef_builtin_expect(self, cond):
        """
        Redefine the macros likely() and unlikely to no-ops, depending on
        condition 'cond'
        """
        self.putln("#if %s" % cond)
        self.putln("    #undef likely")
        self.putln("    #undef unlikely")
        self.putln("    #define likely(x)   (x)")
        self.putln("    #define unlikely(x) (x)")
        self.putln("#endif")

    def redef_builtin_expect(self, cond):
        self.putln("#if %s" % cond)
        self.putln("    #undef likely")
        self.putln("    #undef unlikely")
        self.putln("    #define likely(x)   __builtin_expect(!!(x), 1)")
        self.putln("    #define unlikely(x) __builtin_expect(!!(x), 0)")
        self.putln("#endif")

2452

2453
class PyrexCodeWriter(object):
William Stein's avatar
William Stein committed
2454 2455 2456 2457
    # f                file      output file
    # level            int       indentation level

    def __init__(self, outfile_name):
2458
        self.f = Utils.open_new_file(outfile_name)
William Stein's avatar
William Stein committed
2459
        self.level = 0
2460

William Stein's avatar
William Stein committed
2461 2462
    def putln(self, code):
        self.f.write("%s%s\n" % (" " * self.level, code))
2463

William Stein's avatar
William Stein committed
2464 2465
    def indent(self):
        self.level += 1
2466

William Stein's avatar
William Stein committed
2467 2468 2469
    def dedent(self):
        self.level -= 1

2470 2471
class PyxCodeWriter(object):
    """
2472 2473 2474
    Can be used for writing out some Cython code. To use the indenter
    functionality, the Cython.Compiler.Importer module will have to be used
    to load the code to support python 2.4
2475 2476
    """

2477
    def __init__(self, buffer=None, indent_level=0, context=None, encoding='ascii'):
2478 2479 2480
        self.buffer = buffer or StringIOTree()
        self.level = indent_level
        self.context = context
2481
        self.encoding = encoding
2482 2483 2484

    def indent(self, levels=1):
        self.level += levels
2485
        return True
2486 2487 2488 2489 2490 2491

    def dedent(self, levels=1):
        self.level -= levels

    def indenter(self, line):
        """
2492 2493 2494 2495 2496 2497 2498 2499 2500 2501
        Instead of

            with pyx_code.indenter("for i in range(10):"):
                pyx_code.putln("print i")

        write

            if pyx_code.indenter("for i in range(10);"):
                pyx_code.putln("print i")
                pyx_code.dedent()
2502 2503
        """
        self.putln(line)
2504 2505
        self.indent()
        return True
2506 2507

    def getvalue(self):
2508
        result = self.buffer.getvalue()
2509
        if isinstance(result, bytes):
2510 2511
            result = result.decode(self.encoding)
        return result
2512 2513 2514 2515 2516 2517 2518 2519 2520 2521 2522 2523 2524 2525 2526 2527 2528 2529 2530 2531 2532 2533 2534 2535 2536 2537

    def putln(self, line, context=None):
        context = context or self.context
        if context:
            line = sub_tempita(line, context)
        self._putln(line)

    def _putln(self, line):
        self.buffer.write("%s%s\n" % (self.level * "    ", line))

    def put_chunk(self, chunk, context=None):
        context = context or self.context
        if context:
            chunk = sub_tempita(chunk, context)

        chunk = textwrap.dedent(chunk)
        for line in chunk.splitlines():
            self._putln(line)

    def insertion_point(self):
        return PyxCodeWriter(self.buffer.insertion_point(), self.level,
                             self.context)

    def named_insertion_point(self, name):
        setattr(self, name, self.insertion_point())

2538 2539

class ClosureTempAllocator(object):
2540
    def __init__(self, klass):
2541 2542 2543 2544 2545 2546 2547 2548 2549 2550
        self.klass = klass
        self.temps_allocated = {}
        self.temps_free = {}
        self.temps_count = 0

    def reset(self):
        for type, cnames in self.temps_allocated.items():
            self.temps_free[type] = list(cnames)

    def allocate_temp(self, type):
2551
        if type not in self.temps_allocated:
2552 2553 2554 2555 2556 2557 2558 2559 2560
            self.temps_allocated[type] = []
            self.temps_free[type] = []
        elif self.temps_free[type]:
            return self.temps_free[type].pop(0)
        cname = '%s%d' % (Naming.codewriter_temp_prefix, self.temps_count)
        self.klass.declare_var(pos=None, name=cname, cname=cname, type=type, is_cdef=True)
        self.temps_allocated[type].append(cname)
        self.temps_count += 1
        return cname