.

wcfs/__init__.py

@@ -104,6 +104,7 @@ class Conn(object):
 class _File(object):
     # .wconn    Conn
     # .foid     hex of ZBigFile root object ID
+    # .size     size of this file
     # .blksize  block size of this file
     # .headf    file object of head/file
     # .pinned   {} blk -> rev   that wcfs already sent us for this file
@@ -258,7 +259,6 @@ def resync(wconn, at):
     wconn.at = at
 
 
-
 # mmap creates file mapping representing file[blk_start +blk_len) data as of wconn.at database state.
 @func(Conn)
 def mmap(wconn, foid, blk_start, blk_len): # -> Mapping
@@ -271,9 +271,13 @@ def mmap(wconn, foid, blk_start, blk_len): # -> Mapping
             f.wconn     = wconn
             f.foid      = foid
             f.headf     = wconn._wc._open("head/bigfile/%s" % (h(foid),), "rb")
-            f.blksize   = os.fstat(f.headf.fileno()).st_blksize
             f.pinned    = {}
             f.mmaps     = []
+
+            _ = os.fstat(f.headf.fileno())
+            f.size      = _.st_size
+            f.blksize   = _.st_blksize
+
             wconn._filetab[foid] = f
 
             # start watching f
@@ -286,7 +290,12 @@ def mmap(wconn, foid, blk_start, blk_len): # -> Mapping
         # XXX relock wconn -> f ?
 
         # create memory with head/f mapping and applied pins
-        mem = mm.map_ro(f.headf.fileno(), blk_start*f.blksize, blk_len*f.blksize)
+        # mmap-in zeros after f.size (else access to memory after file.size will raise SIGBUS)
+        start = blk_start*f.blksize
+        mem = mm.map_ro(f.headf.fileno(), start, blk_len*f.blksize)
+        zmemtail = mem[max(f.size, start) - start:]
+        if len(zmemtail) != 0:
+            mm.map_zero_into_ro(zmemtail)
         mmap = _Mapping()
         mmap.file      = f
         mmap.blk_start = blk_start

wcfs/internal/mm.pyx

@@ -101,7 +101,8 @@ def unlock(const unsigned char[::1] mem not None):
 
 from posix.types cimport off_t
 
-# XXX PROT_READ | MAP_SHARED
+# map_ro memory-maps fd[offset +size) as read-only.
+# The mapping is created with MAP_SHARED.
 def map_ro(int fd, off_t offset, size_t size):
     cdef void *addr
 
@@ -111,12 +112,13 @@ def map_ro(int fd, off_t offset, size_t size):
 
     return <unsigned char[:size:1]>addr
 
-# map_into mmaps fd[offset:...] into mem's memory.
+# map_into_ro is similar to map_ro, but mmaps fd[offset:...] into mem's memory.
 def map_into_ro(unsigned char[::1] mem not None, int fd, off_t offset):
     cdef void   *addr = &mem[0]
     cdef size_t size  = mem.shape[0]
 
-    addr = mman.mmap(addr, size, mman.PROT_READ, mman.MAP_SHARED | mman.MAP_FIXED, fd, offset)
+    addr = mman.mmap(addr, size, mman.PROT_READ, mman.MAP_FIXED |
+                                                 mman.MAP_SHARED, fd, offset)
     if addr == mman.MAP_FAILED:
         PyErr_SetFromErrno(OSError)
 
@@ -133,6 +135,36 @@ def unmap(const unsigned char[::1] mem not None):
 
     return
 
+# map_zero_ro creats new read-only mmaping that all reads as zero.
+# created mapping, even after it is accessed, does not consume memory.
+def map_zero_ro(size_t size):
+    cdef void *addr
+    # mmap /dev/zero with MAP_NORESERVE and MAP_SHARED
+    # this way the mapping will be able to be read, but no memory will be allocated to keep it.
+    f = open("/dev/zero", "rb")
+    addr = mman.mmap(NULL, size, mman.PROT_READ, mman.MAP_SHARED | mman.MAP_NORESERVE, f.fileno(), 0)
+    f.close()
+    if addr == mman.MAP_FAILED:
+        PyErr_SetFromErrno(OSError)
+        return
+
+    return <unsigned char[:size:1]>addr
+
+# map_zero_into_ro is similar to map_zero_ro, but mmaps zeros into mem's memory.
+def map_zero_into_ro(unsigned char[::1] mem not None):
+    cdef void   *addr = &mem[0]
+    cdef size_t size  = mem.shape[0]
+
+    f = open("/dev/zero", "rb")
+    addr = mman.mmap(addr, size, mman.PROT_READ, mman.MAP_FIXED |
+                                                 mman.MAP_SHARED | mman.MAP_NORESERVE, f.fileno(), 0)
+    f.close()
+    if addr == mman.MAP_FAILED:
+        PyErr_SetFromErrno(OSError)
+        return
+
+    return
+
 
 # advise advises kernel about use of mem's memory.
 #

wcfs/wcfs_test.py

@@ -1755,8 +1755,7 @@ def test_wcfspy_virtmem():
     assert f.pinned == {2:at1}
     tm1.assertBlk(3, 'd1')
     assert f.pinned == {2:at1}
-    # XXX this (correctly) gets SIGBUS - virtmem should override pages after file.size
-    #tm1.assertBlk(4, '')
+    tm1.assertBlk(4, '')
     assert f.pinned == {2:at1}
 
     at3 = t.commit(zf, {3:'d3'})
@@ -1780,6 +1779,7 @@ def test_wcfspy_virtmem():
 
 
 
+    # XXX mmap after .size completely (start > size)
 
     # XXX w mapping with RW - in sync