X Fixes so that it compiles from scratch; tweaks to weakref

Still trying to see what is going on

Makefile

@@ -34,7 +34,8 @@ ifeq ($(CC),)
 $(error "Cannot defermine py-CC")
 endif
 
-all	: bigfile/_bigfile.so wcfs/wcfs
+#all	: bigfile/_bigfile.so wcfs/wcfs
+all	: bigfile/_bigfile.so
 
 
 ccan_config := 3rdparty/ccan/config.h

bigfile/_file_zodb.pyx

@@ -30,9 +30,20 @@ It provides wcfs integration for ZBigFile handles opened with _use_wcfs=True.
 
 from __future__ import print_function, absolute_import
 
-cdef extern from "wcfs/client/wcfs.h":
+cdef extern from *:
+    """
+    // XXX hack: C++ does not have __builtin_types_compatible_p, but CCAN configure
+    // thinks it does because CCAN is configured via C, not C++.
+    #include <config.h>
+    #undef  HAVE_BUILTIN_TYPES_COMPATIBLE_P
+    #define HAVE_BUILTIN_TYPES_COMPATIBLE_P 0
+    #include <ccan/array_size/array_size.h>
+    """
     pass
 
+#cdef extern from "wcfs/client/wcfs.h":
+#    pass
+
 cdef extern from "bigfile/_bigfile.h":
     struct PyBigFile:
         pass
@@ -46,12 +57,13 @@ cdef extern from "<wendelin/bigfile/file.h>" nogil:
 cdef extern from * nogil:
     """
     extern const bigfile_ops ZBigFile_mmap_ops;
+
     """
     const bigfile_ops ZBigFile_mmap_ops
 
 import wcfs as pywcfs
 from wendelin.lib import zodb as pyzodb
-from wcfs.client cimport _wcfs as wcfs
+#from wcfs.client cimport _wcfs as wcfs
 from golang cimport error, nil, pyerror
 from cpython cimport PyCapsule_New
 
@@ -73,7 +85,7 @@ import weakref
 # - provides blkmmapper with WCFS integration.
 cdef public class _ZBigFile(BigFile) [object _ZBigFile, type _ZBigFile_Type]:
     cdef object     zself   # reference to ZBigFile
-    cdef wcfs.FileH wfileh  # WCFS file handle. Initially nil, opened by blkmmapper
+#   cdef wcfs.FileH wfileh  # WCFS file handle. Initially nil, opened by blkmmapper
 
     # _new creates new _ZBigFile associated with ZBigFile zself.
     # XXX Cython does not allow __new__ nor to change arguments passed to __cinit__ / __init__
@@ -81,14 +93,14 @@ cdef public class _ZBigFile(BigFile) [object _ZBigFile, type _ZBigFile_Type]:
     def _new(zself, blksize):
         cdef _ZBigFile obj = _ZBigFile.__new__(_ZBigFile, blksize)
         obj.zself  = zself
-        obj.wfileh = nil
+#       obj.wfileh = nil
         return obj
 
     def __dealloc__(_ZBigFile zf):
         cdef error err = nil
-        if zf.wfileh != nil:
-            err = zf.wfileh.close()
-        zf.wfileh = nil
+#       if zf.wfileh != nil:
+#           err = zf.wfileh.close()
+#       zf.wfileh = nil
         if err != nil:
             raise pyerror.from_error(err)
 
@@ -107,9 +119,11 @@ cdef public class _ZBigFile(BigFile) [object _ZBigFile, type _ZBigFile_Type]:
     # corresponding to ZBigFile is opened if use_wcfs=True.
     def fileh_open(_ZBigFile zf, bint use_wcfs):
         mmap_overlay = False
-        cdef wcfs.PyFileH pywfileh
+#       cdef wcfs.PyFileH pywfileh
 
         if use_wcfs:
+            1/0
+            """
             mmap_overlay = True
             if zf.wfileh == nil:
                 zconn = zf.zself._p_jar
@@ -119,10 +133,12 @@ cdef public class _ZBigFile(BigFile) [object _ZBigFile, type _ZBigFile_Type]:
                 pywconn   = pywconnOf(zconn)
                 pywfileh  = pywconn.open(zf.zself._p_oid)
                 zf.wfileh = pywfileh.wfileh
+            """
 
         return super(_ZBigFile, zf).fileh_open(mmap_overlay)
 
 
+"""
 # pywconnOf establishes and returns (py) wcfs.Conn associated with zconn.
 #
 # returned wcfs.Conn will be maintained to keep in sync with zconn, and will be
@@ -181,3 +197,4 @@ class ZSync:
     def on_connection_resync(zsync):
         zconn = zsync.zconn_ref()
         zsync.wconn.resync(zconn_at(zconn))
+"""

bigfile/file_zodb.cpp

@@ -20,13 +20,21 @@
 // File file_zodb.cpp provides blkmmapper functions for _ZBigFile.
 // MMapping is implemented via wcfs client.
 
-#include "wcfs/client/wcfs.h"
+// XXX hack: C++ does not have __builtin_types_compatible_p, but CCAN configure
+// thinks it does because CCAN is configured via C, not C++.
+#include <config.h>
+#undef  HAVE_BUILTIN_TYPES_COMPATIBLE_P
+#define HAVE_BUILTIN_TYPES_COMPATIBLE_P 0
+#include <ccan/array_size/array_size.h>
+
+//#include "wcfs/client/wcfs.h"
 #include "wendelin/bigfile/file.h"
 #include "wendelin/bigfile/virtmem.h"
 #include "bigfile/_bigfile.h"
 #include "bigfile/_file_zodb.h"
 #include <ccan/container_of/container_of.h>
 
+/*
 static int zfile_mmap_setup_read(VMA *vma, BigFile *file, blk_t blk, size_t blklen) {
     _ZBigFile* _zfile = container_of(file, _ZBigFile, __pyx_base.file);
 
@@ -80,6 +88,7 @@ static int zfile_munmap(VMA *vma, BigFile *file) {
 
     return 0;
 }
+*/
 
 
 // NOTE reusing whole bigfile_ops for just .mmap* ops.
@@ -87,9 +96,9 @@ extern const bigfile_ops ZBigFile_mmap_ops;
 static bigfile_ops _mkZBigFile_mmap_ops() {
     // workaround for "sorry, unimplemented: non-trivial designated initializers not supported"
     bigfile_ops _;
-    _.mmap_setup_read   = zfile_mmap_setup_read;
-    _.remmap_blk_read   = zfile_remmap_blk_read;
-    _.munmap            = zfile_munmap;
+    _.mmap_setup_read   = NULL;	// zfile_mmap_setup_read;
+    _.remmap_blk_read   = NULL;	// zfile_remmap_blk_read;
+    _.munmap            = NULL;	//zfile_munmap;
     _.loadblk  = NULL;
     _.storeblk = NULL;
     return _;

setup.py

@@ -290,11 +290,13 @@ setup(
                     define_macros       = [('_GNU_SOURCE',None), ('BUILDING_LIBVIRTMEM',None)],
                     language = 'c'),
 
-                   DSO('wendelin.wcfs.client.libwcfs',
-                    ['wcfs/client/wcfs.cpp',
-                     'wcfs/client/wcfs_watchlink.cpp',
-                     'wcfs/client/wcfs_misc.cpp'],
-                    dsos = ['wendelin.bigfile.libvirtmem'])],
+                   #DSO('wendelin.wcfs.client.libwcfs',
+                   # ['wcfs/client/wcfs.cpp',
+                   #  'wcfs/client/wcfs_watchlink.cpp',
+                   #  'wcfs/client/wcfs_misc.cpp'],
+                   # dsos = ['wendelin.bigfile.libvirtmem'])
+
+                   ],
 
     ext_modules = [
                     PyGoExt('wendelin.bigfile._bigfile',
@@ -306,20 +308,21 @@ setup(
                     PyGoExt('wendelin.bigfile._file_zodb',
                         ['bigfile/_file_zodb.pyx',
                          'bigfile/file_zodb.cpp'],
-                        dsos = ['wendelin.wcfs.client.libwcfs']),
+                        dsos = []),
+                        #dsos = ['wendelin.wcfs.client.libwcfs']),
 
-                    PyGoExt('wendelin.wcfs.client._wcfs',
-                        ['wcfs/client/_wcfs.pyx'],
-                        dsos = ['wendelin.wcfs.client.libwcfs']),
+                    #PyGoExt('wendelin.wcfs.client._wcfs',
+                    #    ['wcfs/client/_wcfs.pyx'],
+                    #    dsos = ['wendelin.wcfs.client.libwcfs']),
 
-                    PyGoExt('wendelin.wcfs.internal.wcfs_test',
-                        ['wcfs/internal/wcfs_test.pyx']),
+                    #PyGoExt('wendelin.wcfs.internal.wcfs_test',
+                    #    ['wcfs/internal/wcfs_test.pyx']),
 
-                    Extension('wendelin.wcfs.internal.io',
-                        ['wcfs/internal/io.pyx']),
+                    #Extension('wendelin.wcfs.internal.io',
+                    #    ['wcfs/internal/io.pyx']),
 
-                    Extension('wendelin.wcfs.internal.mm',
-                        ['wcfs/internal/mm.pyx']),
+                    #Extension('wendelin.wcfs.internal.mm',
+                    #    ['wcfs/internal/mm.pyx']),
                   ],
 
     package_dir = {'wendelin': ''},

wcfs/gogccrash

@@ -5,6 +5,7 @@ export GOTRACEBACK=crash
 ulimit -c unlimited
 
 CGO_ENABLED=0 go test -mod=vendor -c
+#CGO_ENABLED=1 go test -mod=vendor -c -race
 #export DBTail_SEED=1602769537289632682
 
 cwd=$(pwd)
@@ -19,6 +20,9 @@ function runtest1() {
 		#GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestZBlk|TestΔBTail' || break
 		#GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestZBlk' || break
 		GOGC=0 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestΔBTail' || break
+		#GOGC=0 GODEBUG=gccheckmark=1 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestΔBTail' || break
+		#GOGC=0 GODEBUG=gcstoptheworld=2 $cwd/wcfs.test -test.v -test.count=100 -test.run 'TestΔBTail' || break
+		#XXX asyncpreemptoff=1
 	done
 }
 

wcfs/vendor/lab.nexedi.com/kirr/neo/go/zodb/internal/weak/weak.go

@@ -28,8 +28,10 @@ import (
 	"fmt"
 	"runtime"
 	"sync"
+	"sync/atomic"
 	"time"
 	"unsafe"
+	"math/rand"
 )
 
 // iface is how Go runtime represents an interface.
@@ -78,8 +80,12 @@ func NewRef(obj interface{}) *Ref {
 		state: objLive,
 	}
 
-	var release func(interface{})
-	release = func(obj interface{}) {
+	runtime.SetFinalizer(obj, w.release)
+	runtime.KeepAlive(obj)
+	return w
+}
+
+func (w *Ref) release(obj interface{}) {
 	ifobj := *(*iface)(unsafe.Pointer(&obj))
 	if w.iface != ifobj {
 		panic(fmt.Sprintf("weak: release: w.iface != obj;  w.iface=%x  obj=%x", w.iface, ifobj))
@@ -94,17 +100,17 @@ func NewRef(obj interface{}) *Ref {
 	w.mu.Lock()
 	if w.state == objGot {
 		w.state = objLive
-			runtime.SetFinalizer(obj, release)
+		runtime.SetFinalizer(obj, w.release)
 	} else {
 		w.state = objReleased
 	}
 	w.mu.Unlock()
-	}
 
-	runtime.SetFinalizer(obj, release)
-	return w
+	runtime.KeepAlive(obj)
 }
 
+var xxx uint64
+
 // Get returns object pointed to by this weak reference.
 //
 // If original object is still alive - it is returned.
@@ -114,11 +120,36 @@ func (w *Ref) Get() (obj interface{}) {
 	if w.state != objReleased {
 		w.state = objGot
 
-		time.Sleep(100*time.Nanosecond)
+		//time.Sleep(100*time.Nanosecond)
 		// recreate interface{} from saved words.
 		// XXX do writes as pointers so that compiler emits write barriers to notify GC?
 		i := (*iface)(unsafe.Pointer(&obj))
-		*i = w.iface
+		//*i = w.iface
+
+		r := rand.Int31n(2)
+		_ = r
+		//if r % 2 == 0 {
+		if true {
+			i.data = w.iface.data
+		} else {
+			i.typ  = w.iface.typ
+		}
+
+		atomic.AddUint64(&xxx, 1) // barrier
+
+		//time.Sleep(100*time.Nanosecond)
+		time.Sleep(100*time.Nanosecond)
+//		go runtime.GC()
+		//time.Sleep(1000*time.Nanosecond)
+	//	time.Sleep(100*time.Nanosecond)
+//		runtime.GC()
+
+		//if r % 2 == 0 {
+		if true {
+			i.typ  = w.iface.typ
+		} else {
+			i.data = w.iface.data
+		}
 	}
 	w.mu.Unlock()