MemoryView.py 29.1 KB
Newer Older
1 2 3 4 5 6 7 8 9 10 11
from __future__ import absolute_import

from .Errors import CompileError, error
from . import ExprNodes
from .ExprNodes import IntNode, NameNode, AttributeNode
from . import Options
from .Code import UtilityCode, TempitaUtilityCode
from .UtilityCode import CythonUtilityCode
from . import Buffer
from . import PyrexTypes
from . import ModuleNode
12

13 14 15
START_ERR = "Start must not be given."
STOP_ERR = "Axis specification only allowed in the 'step' slot."
STEP_ERR = "Step must be omitted, 1, or a valid specifier."
16 17
BOTH_CF_ERR = "Cannot specify an array that is both C and Fortran contiguous."
INVALID_ERR = "Invalid axis specification."
18
NOT_CIMPORTED_ERR = "Variable was not cimported from cython.view"
19
EXPR_ERR = "no expressions allowed in axis spec, only names and literals."
20
CF_ERR = "Invalid axis specification for a C/Fortran contiguous array."
21 22 23
ERR_UNINITIALIZED = ("Cannot check if memoryview %s is initialized without the "
                     "GIL, consider using initializedcheck(False)")

24

Mark Florisson's avatar
Mark Florisson committed
25 26 27
def concat_flags(*flags):
    return "(%s)" % "|".join(flags)

28

Mark Florisson's avatar
Mark Florisson committed
29 30
format_flag = "PyBUF_FORMAT"

31 32 33 34 35 36
memview_c_contiguous = "(PyBUF_C_CONTIGUOUS | PyBUF_FORMAT)"
memview_f_contiguous = "(PyBUF_F_CONTIGUOUS | PyBUF_FORMAT)"
memview_any_contiguous = "(PyBUF_ANY_CONTIGUOUS | PyBUF_FORMAT)"
memview_full_access = "PyBUF_FULL_RO"
#memview_strided_access = "PyBUF_STRIDED_RO"
memview_strided_access = "PyBUF_RECORDS_RO"
37

38 39 40 41 42 43
MEMVIEW_DIRECT = '__Pyx_MEMVIEW_DIRECT'
MEMVIEW_PTR    = '__Pyx_MEMVIEW_PTR'
MEMVIEW_FULL   = '__Pyx_MEMVIEW_FULL'
MEMVIEW_CONTIG = '__Pyx_MEMVIEW_CONTIG'
MEMVIEW_STRIDED= '__Pyx_MEMVIEW_STRIDED'
MEMVIEW_FOLLOW = '__Pyx_MEMVIEW_FOLLOW'
44 45

_spec_to_const = {
46 47 48
        'direct' : MEMVIEW_DIRECT,
        'ptr'    : MEMVIEW_PTR,
        'full'   : MEMVIEW_FULL,
49 50 51 52 53
        'contig' : MEMVIEW_CONTIG,
        'strided': MEMVIEW_STRIDED,
        'follow' : MEMVIEW_FOLLOW,
        }

54 55 56 57 58 59 60 61 62
_spec_to_abbrev = {
    'direct'  : 'd',
    'ptr'     : 'p',
    'full'    : 'f',
    'contig'  : 'c',
    'strided' : 's',
    'follow'  : '_',
}

Mark Florisson's avatar
Mark Florisson committed
63 64
memslice_entry_init = "{ 0, 0, { 0 }, { 0 }, { 0 } }"

65 66 67 68
memview_name = u'memoryview'
memview_typeptr_cname = '__pyx_memoryview_type'
memview_objstruct_cname = '__pyx_memoryview_obj'
memviewslice_cname = u'__Pyx_memviewslice'
Mark Florisson's avatar
Mark Florisson committed
69

70

71 72
def put_init_entry(mv_cname, code):
    code.putln("%s.data = NULL;" % mv_cname)
73 74
    code.putln("%s.memview = NULL;" % mv_cname)

75

76 77
#def axes_to_str(axes):
#    return "".join([access[0].upper()+packing[0] for (access, packing) in axes])
78

79

80
def put_acquire_memoryviewslice(lhs_cname, lhs_type, lhs_pos, rhs, code,
81 82
                                have_gil=False, first_assignment=True):
    "We can avoid decreffing the lhs if we know it is the first assignment"
83 84
    assert rhs.type.is_memoryviewslice

85
    pretty_rhs = rhs.result_in_temp() or rhs.is_simple()
86 87 88 89 90 91
    if pretty_rhs:
        rhstmp = rhs.result()
    else:
        rhstmp = code.funcstate.allocate_temp(lhs_type, manage_ref=False)
        code.putln("%s = %s;" % (rhstmp, rhs.result_as(lhs_type)))

92 93
    # Allow uninitialized assignment
    #code.putln(code.put_error_if_unbound(lhs_pos, rhs.entry))
94
    put_assign_to_memviewslice(lhs_cname, rhs, rhstmp, lhs_type, code,
95
                               have_gil=have_gil, first_assignment=first_assignment)
96 97 98

    if not pretty_rhs:
        code.funcstate.release_temp(rhstmp)
99

100

101
def put_assign_to_memviewslice(lhs_cname, rhs, rhs_cname, memviewslicetype, code,
102 103 104 105
                               have_gil=False, first_assignment=False):
    if not first_assignment:
        code.put_xdecref_memoryviewslice(lhs_cname, have_gil=have_gil)

106 107
    if not rhs.result_in_temp():
        rhs.make_owned_memoryviewslice(code)
108

109 110
    code.putln("%s = %s;" % (lhs_cname, rhs_cname))

111

112
def get_buf_flags(specs):
113 114 115 116 117 118 119 120 121 122 123 124 125 126
    is_c_contig, is_f_contig = is_cf_contig(specs)

    if is_c_contig:
        return memview_c_contiguous
    elif is_f_contig:
        return memview_f_contiguous

    access, packing = zip(*specs)

    if 'full' in access or 'ptr' in access:
        return memview_full_access
    else:
        return memview_strided_access

127

128 129 130 131 132
def insert_newaxes(memoryviewtype, n):
    axes = [('direct', 'strided')] * n
    axes.extend(memoryviewtype.axes)
    return PyrexTypes.MemoryViewSliceType(memoryviewtype.dtype, axes)

133

134 135 136 137 138 139
def broadcast_types(src, dst):
    n = abs(src.ndim - dst.ndim)
    if src.ndim < dst.ndim:
        return insert_newaxes(src, n), dst
    else:
        return src, insert_newaxes(dst, n)
140

141

142
def valid_memslice_dtype(dtype, i=0):
143 144
    """
    Return whether type dtype can be used as the base type of a
145 146 147
    memoryview slice.

    We support structs, numeric types and objects
148 149 150 151
    """
    if dtype.is_complex and dtype.real_type.is_int:
        return False

152 153 154
    if dtype is PyrexTypes.c_bint_type:
        return False

155 156 157 158 159 160 161
    if dtype.is_struct and dtype.kind == 'struct':
        for member in dtype.scope.var_entries:
            if not valid_memslice_dtype(member.type):
                return False

        return True

162 163
    return (
        dtype.is_error or
164 165
        # Pointers are not valid (yet)
        # (dtype.is_ptr and valid_memslice_dtype(dtype.base_type)) or
166 167
        (dtype.is_array and i < 8 and
         valid_memslice_dtype(dtype.base_type, i + 1)) or
168 169
        dtype.is_numeric or
        dtype.is_pyobject or
170
        dtype.is_fused or # accept this as it will be replaced by specializations later
171 172 173
        (dtype.is_typedef and valid_memslice_dtype(dtype.typedef_base_type))
    )

174

175
class MemoryViewSliceBufferEntry(Buffer.BufferEntry):
176 177 178 179
    """
    May be used during code generation time to be queried for
    shape/strides/suboffsets attributes, or to perform indexing or slicing.
    """
180 181 182 183
    def __init__(self, entry):
        self.entry = entry
        self.type = entry.type
        self.cname = entry.cname
184

185 186 187
        self.buf_ptr = "%s.data" % self.cname

        dtype = self.entry.type.dtype
188 189
        self.buf_ptr_type = PyrexTypes.CPtrType(dtype)
        self.init_attributes()
190 191

    def get_buf_suboffsetvars(self):
Mark Florisson's avatar
Mark Florisson committed
192
        return self._for_all_ndim("%s.suboffsets[%d]")
193 194 195 196 197 198 199

    def get_buf_stridevars(self):
        return self._for_all_ndim("%s.strides[%d]")

    def get_buf_shapevars(self):
        return self._for_all_ndim("%s.shape[%d]")

200
    def generate_buffer_lookup_code(self, code, index_cnames):
201 202 203 204 205
        axes = [(dim, index_cnames[dim], access, packing)
                    for dim, (access, packing) in enumerate(self.type.axes)]
        return self._generate_buffer_lookup_code(code, axes)

    def _generate_buffer_lookup_code(self, code, axes, cast_result=True):
206 207 208 209
        """
        Generate a single expression that indexes the memory view slice
        in each dimension.
        """
210
        bufp = self.buf_ptr
211
        type_decl = self.type.dtype.empty_declaration_code()
212

213
        for dim, index, access, packing in axes:
214 215 216 217
            shape = "%s.shape[%d]" % (self.cname, dim)
            stride = "%s.strides[%d]" % (self.cname, dim)
            suboffset = "%s.suboffsets[%d]" % (self.cname, dim)

218 219
            flag = get_memoryview_flag(access, packing)

220 221 222 223 224
            if flag in ("generic", "generic_contiguous"):
                # Note: we cannot do cast tricks to avoid stride multiplication
                #       for generic_contiguous, as we may have to do (dtype *)
                #       or (dtype **) arithmetic, we won't know which unless
                #       we check suboffsets
225
                code.globalstate.use_utility_code(memviewslice_index_helpers)
226
                bufp = ('__pyx_memviewslice_index_full(%s, %s, %s, %s)' %
227 228
                                            (bufp, index, stride, suboffset))

229
            elif flag == "indirect":
230 231
                bufp = "(%s + %s * %s)" % (bufp, index, stride)
                bufp = ("(*((char **) %s) + %s)" % (bufp, suboffset))
232

233
            elif flag == "indirect_contiguous":
234 235
                # Note: we do char ** arithmetic
                bufp = "(*((char **) %s + %s) + %s)" % (bufp, index, suboffset)
236

237
            elif flag == "strided":
238 239 240
                bufp = "(%s + %s * %s)" % (bufp, index, stride)

            else:
241
                assert flag == 'contiguous', flag
242 243 244 245
                bufp = '((char *) (((%s *) %s) + %s))' % (type_decl, bufp, index)

            bufp = '( /* dim=%d */ %s )' % (dim, bufp)

246 247 248 249 250
        if cast_result:
            return "((%s *) %s)" % (type_decl, bufp)

        return bufp

251
    def generate_buffer_slice_code(self, code, indices, dst, have_gil,
252
                                   have_slices, directives):
253 254 255
        """
        Slice a memoryviewslice.

256 257
        indices     - list of index nodes. If not a SliceNode, or NoneNode,
                      then it must be coercible to Py_ssize_t
258 259

        Simply call __pyx_memoryview_slice_memviewslice with the right
260 261 262
        arguments, unless the dimension is omitted or a bare ':', in which
        case we copy over the shape/strides/suboffsets attributes directly
        for that dimension.
263
        """
264 265 266 267
        src = self.cname

        code.putln("%(dst)s.data = %(src)s.data;" % locals())
        code.putln("%(dst)s.memview = %(src)s.memview;" % locals())
268
        code.put_incref_memoryviewslice(dst)
269

270 271 272 273
        all_dimensions_direct = all(access == 'direct' for access, packing in self.type.axes)
        suboffset_dim_temp = []

        def get_suboffset_dim():
Stefan Behnel's avatar
Stefan Behnel committed
274
            # create global temp variable at request
275 276 277 278 279 280
            if not suboffset_dim_temp:
                suboffset_dim = code.funcstate.allocate_temp(PyrexTypes.c_int_type, manage_ref=False)
                code.putln("%s = -1;" % suboffset_dim)
                suboffset_dim_temp.append(suboffset_dim)
            return suboffset_dim_temp[0]

281
        dim = -1
282
        new_ndim = 0
283
        for index in indices:
284 285 286 287 288 289 290 291 292 293
            if index.is_none:
                # newaxis
                for attrib, value in [('shape', 1), ('strides', 0), ('suboffsets', -1)]:
                    code.putln("%s.%s[%d] = %d;" % (dst, attrib, new_ndim, value))

                new_ndim += 1
                continue

            dim += 1
            access, packing = self.type.axes[dim]
294
            error_goto = code.error_goto(index.pos)
295

296
            if isinstance(index, ExprNodes.SliceNode):
297
                # slice, unspecified dimension, or part of ellipsis
Stefan Behnel's avatar
Stefan Behnel committed
298
                d = dict(locals())
299 300 301
                for s in "start stop step".split():
                    idx = getattr(index, s)
                    have_idx = d['have_' + s] = not idx.is_none
302 303 304
                    d[s] = idx.result() if have_idx else "0"

                if not (d['have_start'] or d['have_stop'] or d['have_step']):
305 306 307
                    # full slice (:), simply copy over the extent, stride
                    # and suboffset. Also update suboffset_dim if needed
                    d['access'] = access
Stefan Behnel's avatar
Stefan Behnel committed
308
                    util_name = "SimpleSlice"
309
                else:
Stefan Behnel's avatar
Stefan Behnel committed
310
                    util_name = "ToughSlice"
311

312
                new_ndim += 1
313 314 315 316
            else:
                # normal index
                idx = index.result()

317 318 319
                indirect = access != 'direct'
                if indirect:
                    generic = access == 'full'
320 321 322 323 324
                    if new_ndim != 0:
                        return error(index.pos,
                                     "All preceding dimensions must be "
                                     "indexed and not sliced")

325 326 327 328 329
                d = dict(
                    locals(),
                    wraparound=int(directives['wraparound']),
                    boundscheck=int(directives['boundscheck'])
                )
Stefan Behnel's avatar
Stefan Behnel committed
330 331 332 333
                util_name = "SliceIndex"

            _, impl = TempitaUtilityCode.load_as_string(util_name, "MemoryView_C.c", context=d)
            code.put(impl)
334

335 336
        if suboffset_dim_temp:
            code.funcstate.release_temp(suboffset_dim_temp[0])
337

338

339 340 341 342 343
def empty_slice(pos):
    none = ExprNodes.NoneNode(pos)
    return ExprNodes.SliceNode(pos, start=none,
                               stop=none, step=none)

344 345

def unellipsify(indices, ndim):
346 347
    result = []
    seen_ellipsis = False
348
    have_slices = False
349

350
    newaxes = [newaxis for newaxis in indices if newaxis.is_none]
351 352
    n_indices = len(indices) - len(newaxes)

353 354
    for index in indices:
        if isinstance(index, ExprNodes.EllipsisNode):
355
            have_slices = True
356
            full_slice = empty_slice(index.pos)
357

358 359 360
            if seen_ellipsis:
                result.append(full_slice)
            else:
361
                nslices = ndim - n_indices + 1
362 363 364
                result.extend([full_slice] * nslices)
                seen_ellipsis = True
        else:
365
            have_slices = have_slices or index.is_slice or index.is_none
366 367
            result.append(index)

368 369
    result_length = len(result) - len(newaxes)
    if result_length < ndim:
370
        have_slices = True
371
        nslices = ndim - result_length
372 373
        result.extend([empty_slice(indices[-1].pos)] * nslices)

374 375
    return have_slices, result, newaxes

376

377 378 379 380 381 382 383 384 385 386 387 388 389 390 391
def get_memoryview_flag(access, packing):
    if access == 'full' and packing in ('strided', 'follow'):
        return 'generic'
    elif access == 'full' and packing == 'contig':
        return 'generic_contiguous'
    elif access == 'ptr' and packing in ('strided', 'follow'):
        return 'indirect'
    elif access == 'ptr' and packing == 'contig':
        return 'indirect_contiguous'
    elif access == 'direct' and packing in ('strided', 'follow'):
        return 'strided'
    else:
        assert (access, packing) == ('direct', 'contig'), (access, packing)
        return 'contiguous'

392

393 394 395
def get_is_contig_func_name(contig_type, ndim):
    assert contig_type in ('C', 'F')
    return "__pyx_memviewslice_is_contig_%s%d" % (contig_type, ndim)
396

397

398 399 400 401
def get_is_contig_utility(contig_type, ndim):
    assert contig_type in ('C', 'F')
    C = dict(context, ndim=ndim, contig_type=contig_type)
    utility = load_memview_c_utility("MemviewSliceCheckContig", C, requires=[is_contig_utility])
402
    return utility
403

Stefan Behnel's avatar
Stefan Behnel committed
404

405
def slice_iter(slice_type, slice_result, ndim, code):
406
    if slice_type.is_c_contig or slice_type.is_f_contig:
407
        return ContigSliceIter(slice_type, slice_result, ndim, code)
408
    else:
409 410
        return StridedSliceIter(slice_type, slice_result, ndim, code)

411 412

class SliceIter(object):
413
    def __init__(self, slice_type, slice_result, ndim, code):
414
        self.slice_type = slice_type
415
        self.slice_result = slice_result
416 417 418
        self.code = code
        self.ndim = ndim

419

420
class ContigSliceIter(SliceIter):
421
    def start_loops(self):
422 423 424
        code = self.code
        code.begin_block()

425
        type_decl = self.slice_type.dtype.empty_declaration_code()
426

427 428
        total_size = ' * '.join("%s.shape[%d]" % (self.slice_result, i)
                                for i in range(self.ndim))
429 430
        code.putln("Py_ssize_t __pyx_temp_extent = %s;" % total_size)
        code.putln("Py_ssize_t __pyx_temp_idx;")
431
        code.putln("%s *__pyx_temp_pointer = (%s *) %s.data;" % (
432
            type_decl, type_decl, self.slice_result))
433 434 435 436 437 438
        code.putln("for (__pyx_temp_idx = 0; "
                        "__pyx_temp_idx < __pyx_temp_extent; "
                        "__pyx_temp_idx++) {")

        return "__pyx_temp_pointer"

439
    def end_loops(self):
440 441 442 443
        self.code.putln("__pyx_temp_pointer += 1;")
        self.code.putln("}")
        self.code.end_block()

444

445
class StridedSliceIter(SliceIter):
446
    def start_loops(self):
447 448 449 450
        code = self.code
        code.begin_block()

        for i in range(self.ndim):
451
            t = i, self.slice_result, i
452 453 454 455 456
            code.putln("Py_ssize_t __pyx_temp_extent_%d = %s.shape[%d];" % t)
            code.putln("Py_ssize_t __pyx_temp_stride_%d = %s.strides[%d];" % t)
            code.putln("char *__pyx_temp_pointer_%d;" % i)
            code.putln("Py_ssize_t __pyx_temp_idx_%d;" % i)

457
        code.putln("__pyx_temp_pointer_0 = %s.data;" % self.slice_result)
458 459 460 461 462 463 464 465 466 467 468

        for i in range(self.ndim):
            if i > 0:
                code.putln("__pyx_temp_pointer_%d = __pyx_temp_pointer_%d;" % (i, i - 1))

            code.putln("for (__pyx_temp_idx_%d = 0; "
                            "__pyx_temp_idx_%d < __pyx_temp_extent_%d; "
                            "__pyx_temp_idx_%d++) {" % (i, i, i, i))

        return "__pyx_temp_pointer_%d" % (self.ndim - 1)

469
    def end_loops(self):
470 471 472 473 474 475
        code = self.code
        for i in range(self.ndim - 1, -1, -1):
            code.putln("__pyx_temp_pointer_%d += __pyx_temp_stride_%d;" % (i, i))
            code.putln("}")

        code.end_block()
476 477


478 479 480 481 482
def copy_c_or_fortran_cname(memview):
    if memview.is_c_contig:
        c_or_f = 'c'
    else:
        c_or_f = 'f'
Kurt Smith's avatar
Kurt Smith committed
483

484 485
    return "__pyx_memoryview_copy_slice_%s_%s" % (
            memview.specialization_suffix(), c_or_f)
Kurt Smith's avatar
Kurt Smith committed
486

487 488 489 490 491 492 493
def get_copy_new_utility(pos, from_memview, to_memview):
    if from_memview.dtype != to_memview.dtype:
        return error(pos, "dtypes must be the same!")
    if len(from_memview.axes) != len(to_memview.axes):
        return error(pos, "number of dimensions must be same")
    if not (to_memview.is_c_contig or to_memview.is_f_contig):
        return error(pos, "to_memview must be c or f contiguous.")
Kurt Smith's avatar
Kurt Smith committed
494

495 496 497 498
    for (access, packing) in from_memview.axes:
        if access != 'direct':
            return error(
                    pos, "cannot handle 'full' or 'ptr' access at this time.")
499

500 501 502 503 504 505 506 507 508 509
    if to_memview.is_c_contig:
        mode = 'c'
        contig_flag = memview_c_contiguous
    elif to_memview.is_f_contig:
        mode = 'fortran'
        contig_flag = memview_f_contiguous

    return load_memview_c_utility(
        "CopyContentsUtility",
        context=dict(
Mark Florisson's avatar
Mark Florisson committed
510
            context,
Mark Florisson's avatar
Mark Florisson committed
511
            mode=mode,
512
            dtype_decl=to_memview.dtype.empty_declaration_code(),
Mark Florisson's avatar
Mark Florisson committed
513
            contig_flag=contig_flag,
514
            ndim=to_memview.ndim,
515 516
            func_cname=copy_c_or_fortran_cname(to_memview),
            dtype_is_object=int(to_memview.dtype.is_pyobject)),
517
        requires=[copy_contents_new_utility])
518

519 520 521 522 523 524 525
def get_axes_specs(env, axes):
    '''
    get_axes_specs(env, axes) -> list of (access, packing) specs for each axis.
    access is one of 'full', 'ptr' or 'direct'
    packing is one of 'contig', 'strided' or 'follow'
    '''

526
    cythonscope = env.global_scope().context.cython_scope
527
    cythonscope.load_cythonscope()
528 529 530 531 532 533 534 535 536 537 538 539 540 541
    viewscope = cythonscope.viewscope

    access_specs = tuple([viewscope.lookup(name)
                    for name in ('full', 'direct', 'ptr')])
    packing_specs = tuple([viewscope.lookup(name)
                    for name in ('contig', 'strided', 'follow')])

    is_f_contig, is_c_contig = False, False
    default_access, default_packing = 'direct', 'strided'
    cf_access, cf_packing = default_access, 'follow'

    axes_specs = []
    # analyse all axes.
    for idx, axis in enumerate(axes):
542
        if not axis.start.is_none:
543 544
            raise CompileError(axis.start.pos,  START_ERR)

545
        if not axis.stop.is_none:
546 547
            raise CompileError(axis.stop.pos, STOP_ERR)

548 549
        if axis.step.is_none:
            axes_specs.append((default_access, default_packing))
550 551

        elif isinstance(axis.step, IntNode):
552
            # the packing for the ::1 axis is contiguous,
553
            # all others are cf_packing.
554 555
            if axis.step.compile_time_value(env) != 1:
                raise CompileError(axis.step.pos, STEP_ERR)
556

557
            axes_specs.append((cf_access, 'cfcontig'))
558

559
        elif isinstance(axis.step, (NameNode, AttributeNode)):
560 561 562
            entry = _get_resolved_spec(env, axis.step)
            if entry.name in view_constant_to_access_packing:
                axes_specs.append(view_constant_to_access_packing[entry.name])
563
            else:
Stefan Behnel's avatar
Stefan Behnel committed
564
                raise CompileError(axis.step.pos, INVALID_ERR)
565 566 567 568

        else:
            raise CompileError(axis.step.pos, INVALID_ERR)

569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628
    # First, find out if we have a ::1 somewhere
    contig_dim = 0
    is_contig = False
    for idx, (access, packing) in enumerate(axes_specs):
        if packing == 'cfcontig':
            if is_contig:
                raise CompileError(axis.step.pos, BOTH_CF_ERR)

            contig_dim = idx
            axes_specs[idx] = (access, 'contig')
            is_contig = True

    if is_contig:
        # We have a ::1 somewhere, see if we're C or Fortran contiguous
        if contig_dim == len(axes) - 1:
            is_c_contig = True
        else:
            is_f_contig = True

            if contig_dim and not axes_specs[contig_dim - 1][0] in ('full', 'ptr'):
                raise CompileError(axes[contig_dim].pos,
                                   "Fortran contiguous specifier must follow an indirect dimension")

        if is_c_contig:
            # Contiguous in the last dimension, find the last indirect dimension
            contig_dim = -1
            for idx, (access, packing) in enumerate(reversed(axes_specs)):
                if access in ('ptr', 'full'):
                    contig_dim = len(axes) - idx - 1

        # Replace 'strided' with 'follow' for any dimension following the last
        # indirect dimension, the first dimension or the dimension following
        # the ::1.
        #               int[::indirect, ::1, :, :]
        #                                    ^  ^
        #               int[::indirect, :, :, ::1]
        #                               ^  ^
        start = contig_dim + 1
        stop = len(axes) - is_c_contig
        for idx, (access, packing) in enumerate(axes_specs[start:stop]):
            idx = contig_dim + 1 + idx
            if access != 'direct':
                raise CompileError(axes[idx].pos,
                                   "Indirect dimension may not follow "
                                   "Fortran contiguous dimension")
            if packing == 'contig':
                raise CompileError(axes[idx].pos,
                                   "Dimension may not be contiguous")
            axes_specs[idx] = (access, cf_packing)

        if is_c_contig:
            # For C contiguity, we need to fix the 'contig' dimension
            # after the loop
            a, p = axes_specs[-1]
            axes_specs[-1] = a, 'contig'

    validate_axes_specs([axis.start.pos for axis in axes],
                        axes_specs,
                        is_c_contig,
                        is_f_contig)
629

630 631
    return axes_specs

632

633 634 635 636 637 638 639 640
def validate_axes(pos, axes):
    if len(axes) >= Options.buffer_max_dims:
        error(pos, "More dimensions than the maximum number"
                   " of buffer dimensions were used.")
        return False

    return True

Mark Florisson's avatar
Mark Florisson committed
641

642
def is_cf_contig(specs):
643 644
    is_c_contig = is_f_contig = False

645
    if len(specs) == 1 and specs == [('direct', 'contig')]:
646
        is_c_contig = True
647

Mark Florisson's avatar
Mark Florisson committed
648
    elif (specs[-1] == ('direct','contig') and
649
          all(axis == ('direct','follow') for axis in specs[:-1])):
650 651 652
        # c_contiguous: 'follow', 'follow', ..., 'follow', 'contig'
        is_c_contig = True

653
    elif (len(specs) > 1 and
Mark Florisson's avatar
Mark Florisson committed
654
        specs[0] == ('direct','contig') and
655
        all(axis == ('direct','follow') for axis in specs[1:])):
656 657 658
        # f_contiguous: 'contig', 'follow', 'follow', ..., 'follow'
        is_f_contig = True

659 660
    return is_c_contig, is_f_contig

661

662 663 664 665 666 667 668 669 670 671 672 673 674 675
def get_mode(specs):
    is_c_contig, is_f_contig = is_cf_contig(specs)

    if is_c_contig:
        return 'c'
    elif is_f_contig:
        return 'fortran'

    for access, packing in specs:
        if access in ('ptr', 'full'):
            return 'full'

    return 'strided'

676 677 678 679 680 681 682 683 684
view_constant_to_access_packing = {
    'generic':              ('full',   'strided'),
    'strided':              ('direct', 'strided'),
    'indirect':             ('ptr',    'strided'),
    'generic_contiguous':   ('full',   'contig'),
    'contiguous':           ('direct', 'contig'),
    'indirect_contiguous':  ('ptr',    'contig'),
}

685
def validate_axes_specs(positions, specs, is_c_contig, is_f_contig):
686 687 688 689

    packing_specs = ('contig', 'strided', 'follow')
    access_specs = ('direct', 'ptr', 'full')

690
    # is_c_contig, is_f_contig = is_cf_contig(specs)
691

692
    has_contig = has_follow = has_strided = has_generic_contig = False
693

694 695 696 697 698
    last_indirect_dimension = -1
    for idx, (access, packing) in enumerate(specs):
        if access == 'ptr':
            last_indirect_dimension = idx

699
    for idx, (pos, (access, packing)) in enumerate(zip(positions, specs)):
700 701 702 703 704 705 706 707 708

        if not (access in access_specs and
                packing in packing_specs):
            raise CompileError(pos, "Invalid axes specification.")

        if packing == 'strided':
            has_strided = True
        elif packing == 'contig':
            if has_contig:
709 710 711 712 713 714 715
                raise CompileError(pos, "Only one direct contiguous "
                                        "axis may be specified.")

            valid_contig_dims = last_indirect_dimension + 1, len(specs) - 1
            if idx not in valid_contig_dims and access != 'ptr':
                if last_indirect_dimension + 1 != len(specs) - 1:
                    dims = "dimensions %d and %d" % valid_contig_dims
716
                else:
717 718 719 720
                    dims = "dimension %d" % valid_contig_dims[0]

                raise CompileError(pos, "Only %s may be contiguous and direct" % dims)

721
            has_contig = access != 'ptr'
722 723 724 725 726 727
        elif packing == 'follow':
            if has_strided:
                raise CompileError(pos, "A memoryview cannot have both follow and strided axis specifiers.")
            if not (is_c_contig or is_f_contig):
                raise CompileError(pos, "Invalid use of the follow specifier.")

728 729 730
        if access in ('ptr', 'full'):
            has_strided = False

731 732 733 734 735 736 737 738 739 740 741 742 743 744
def _get_resolved_spec(env, spec):
    # spec must be a NameNode or an AttributeNode
    if isinstance(spec, NameNode):
        return _resolve_NameNode(env, spec)
    elif isinstance(spec, AttributeNode):
        return _resolve_AttributeNode(env, spec)
    else:
        raise CompileError(spec.pos, INVALID_ERR)

def _resolve_NameNode(env, node):
    try:
        resolved_name = env.lookup(node.name).name
    except AttributeError:
        raise CompileError(node.pos, INVALID_ERR)
745

746
    viewscope = env.global_scope().context.cython_scope.viewscope
747 748 749 750 751
    entry = viewscope.lookup(resolved_name)
    if entry is None:
        raise CompileError(node.pos, NOT_CIMPORTED_ERR)

    return entry
752 753 754 755 756 757 758 759 760 761 762 763 764

def _resolve_AttributeNode(env, node):
    path = []
    while isinstance(node, AttributeNode):
        path.insert(0, node.attribute)
        node = node.obj
    if isinstance(node, NameNode):
        path.insert(0, node.name)
    else:
        raise CompileError(node.pos, EXPR_ERR)
    modnames = path[:-1]
    # must be at least 1 module name, o/w not an AttributeNode.
    assert modnames
765 766 767 768

    scope = env
    for modname in modnames:
        mod = scope.lookup(modname)
769
        if not mod or not mod.as_module:
770 771 772 773
            raise CompileError(
                    node.pos, "undeclared name not builtin: %s" % modname)
        scope = mod.as_module

774 775 776 777 778
    entry = scope.lookup(path[-1])
    if not entry:
        raise CompileError(node.pos, "No such attribute '%s'" % path[-1])

    return entry
779

780 781 782 783
#
### Utility loading
#

784 785 786
def load_memview_cy_utility(util_code_name, context=None, **kwargs):
    return CythonUtilityCode.load(util_code_name, "MemoryView.pyx",
                                  context=context, **kwargs)
787

788
def load_memview_c_utility(util_code_name, context=None, **kwargs):
789 790 791 792 793
    if context is None:
        return UtilityCode.load(util_code_name, "MemoryView_C.c", **kwargs)
    else:
        return TempitaUtilityCode.load(util_code_name, "MemoryView_C.c",
                                       context=context, **kwargs)
794

795
def use_cython_array_utility_code(env):
796 797 798
    cython_scope = env.global_scope().context.cython_scope
    cython_scope.load_cythonscope()
    cython_scope.viewscope.lookup('array_cwrapper').used = True
799

Mark Florisson's avatar
Mark Florisson committed
800
context = {
801 802 803
    'memview_struct_name': memview_objstruct_cname,
    'max_dims': Options.buffer_max_dims,
    'memviewslice_name': memviewslice_cname,
Mark Florisson's avatar
Mark Florisson committed
804
    'memslice_init': memslice_entry_init,
805
}
806
memviewslice_declare_code = load_memview_c_utility(
807
        "MemviewSliceStruct",
808 809
        context=context,
        requires=[])
810

811
atomic_utility = load_memview_c_utility("Atomics", context)
812

813
memviewslice_init_code = load_memview_c_utility(
814
    "MemviewSliceInit",
815
    context=dict(context, BUF_MAX_NDIMS=Options.buffer_max_dims),
816
    requires=[memviewslice_declare_code,
817
              atomic_utility],
818 819 820 821 822 823 824
)

memviewslice_index_helpers = load_memview_c_utility("MemviewSliceIndex")

typeinfo_to_format_code = load_memview_cy_utility(
        "BufferFormatFromTypeInfo", requires=[Buffer._typeinfo_to_format_code])

825
is_contig_utility = load_memview_c_utility("MemviewSliceIsContig", context)
826
overlapping_utility = load_memview_c_utility("OverlappingSlices", context)
827 828 829 830 831
copy_contents_new_utility = load_memview_c_utility(
    "MemviewSliceCopyTemplate",
    context,
    requires=[], # require cython_array_utility_code
)
832

833 834 835 836 837
view_utility_code = load_memview_cy_utility(
        "View.MemoryView",
        context=context,
        requires=[Buffer.GetAndReleaseBufferUtilityCode(),
                  Buffer.buffer_struct_declare_code,
838
                  Buffer.buffer_formats_declare_code,
839 840 841
                  memviewslice_init_code,
                  is_contig_utility,
                  overlapping_utility,
842 843
                  copy_contents_new_utility,
                  ModuleNode.capsule_utility_code],
844
)
845 846 847
view_utility_whitelist = ('array', 'memoryview', 'array_cwrapper',
                          'generic', 'strided', 'indirect', 'contiguous',
                          'indirect_contiguous')
848

849
memviewslice_declare_code.requires.append(view_utility_code)
850
copy_contents_new_utility.requires.append(view_utility_code)