wcfs: Initial stub

Add initial stub for WCFS program and tests.
WCFS functionality will be added step-by-step in follow-up commits.

Some preliminary history:

0ae88a32       X .nxdtest: Verify Go bits with GOMAXPROCS=1,2,`nproc`
23528eb4       X wcfs: make it to use go modules for dependencies

.nxdtest

@@ -3,6 +3,10 @@
 
 storv = ['fs', 'zeo', 'neo'] # storage backends to test against
 
+# some bugs are only likely to trigger when there is only 1 or 2 main OS thread(s) in wcfs
+# GOMAXPROCS='' means use `nproc`
+gonprocv = ['1', '2', '']           # GOMAXPROCS=... to test against
+
 
 # test.t & friends unit-test core of UVMM and are Go-, Python- and ZODB-storage independent.
 # we don't run test.vg* because there is currently no valgrind on SlapOS.
@@ -27,3 +31,17 @@ for stor in storv:
     TestCase('test.py/%s-!wcfs' % stor, ['make', 'test.py'],
              envadj={'WENDELIN_CORE_TEST_DB': '<%s>' % stor},
              summaryf=PyTest.summary)
+
+
+# test.go unit-tests Go bits in wcfs.
+for nproc in gonprocv:
+    TestCase('test.go:%s' % nproc, ['make', 'test.go'],
+             envadj={'GOMAXPROCS': nproc})
+
+# test.wcfs/<stor> runs unit tests for WCFS
+for stor in storv:
+    envdb = {'WENDELIN_CORE_TEST_DB': '<%s>' % stor}
+
+    for nproc in gonprocv:
+        TestCase('test.wcfs/%s:%s' % (stor, nproc), ['make', 'test.wcfs'],
+                 envadj=dict(GOMAXPROCS=nproc, **envdb), summaryf=PyTest.summary)

Makefile

@@ -23,6 +23,7 @@ PYTHON	?= python
 PYTEST	?= $(PYTHON) -m pytest
 PYBENCH ?= $(PYTHON) -m golang.cmd.pybench
 VALGRIND?= valgrind
+GO	?= go
 
 # use the same C compiler as python
 # (for example it could be `gcc -m64` on a 32bit userspace)
@@ -33,7 +34,7 @@ ifeq ($(CC),)
 $(error "Cannot defermine py-CC")
 endif
 
-all	: bigfile/_bigfile.so
+all	: bigfile/_bigfile.so wcfs/wcfs
 
 
 ccan_config := 3rdparty/ccan/config.h
@@ -42,6 +43,9 @@ bigfile/_bigfile.so : $(ccan_config) FORCE
 	$(PYTHON) setup.py build_dso --inplace
 	$(PYTHON) setup.py ll_build_ext --inplace
 
+wcfs/wcfs: FORCE
+	cd wcfs && $(GO) build
+
 
 FORCE	:
 
@@ -69,7 +73,7 @@ CFLAGS	:= -g -Wall -D_GNU_SOURCE -std=gnu99 -fplan9-extensions	\
 # XXX hack ugly
 LOADLIBES=lib/bug.c lib/utils.c 3rdparty/ccan/ccan/tap/tap.c
 TESTS	:= $(patsubst %.c,%,$(wildcard bigfile/tests/test_*.c))
-test	: test.t test.py test.fault test.asan test.tsan test.vgmem test.vghel test.vgdrd
+test	: test.t test.go test.wcfs test.py test.fault test.asan test.tsan test.vgmem test.vghel test.vgdrd
 
 # TODO move XFAIL markers into *.c
 
@@ -165,17 +169,25 @@ test.vgdrd: $(TESTS:%=%.vgdrdrun)
 
 # run python tests
 PYTEST_IGNORE	:=  --ignore=3rdparty --ignore=build --ignore=t
-test.py	: bigfile/_bigfile.so
-	$(PYTEST) $(PYTEST_IGNORE)
+# wcfs unit-tests
+test.wcfs : bigfile/_bigfile.so wcfs/wcfs
+	$(PYTEST) wcfs/
+# unit/functional tests for whole wendelin.core
+test.py	: bigfile/_bigfile.so wcfs/wcfs
+	$(PYTEST) $(PYTEST_IGNORE) --ignore=wcfs
 
 # test.py via Valgrind (very slow)
-test.py.vghel: bigfile/_bigfile.so
+test.py.vghel: bigfile/_bigfile.so wcfs/wcfs
 	$(call vgxrun,--tool=helgrind, $(PYTEST) $(PYTEST_IGNORE))
 
-test.py.drd: bigfile/_bigfile.so
+test.py.drd: bigfile/_bigfile.so wcfs/wcfs
 	$(call vgxrun,--tool=drd, $(PYTEST) $(PYTEST_IGNORE))
 
 
+# run go tests
+test.go	:
+	cd wcfs && $(GO) test -count=1 ./... # -count=1 disables tests caching
+
 
 # test pagefault for double/real faults - it should crash
 tfault	:= bigfile/tests/tfault
@@ -194,5 +206,5 @@ bench	: bench.t bench.py
 
 bench.t	: $(BENCHV.C:%=%.trun)
 
-bench.py: bigfile/_bigfile.so
+bench.py: bigfile/_bigfile.so wcfs/wcfs
 	$(PYBENCH) --count=3 --forked $(PYTEST_IGNORE)

setup.py

@@ -21,11 +21,13 @@ from golang.pyx.build import setup, DSO as _DSO, Extension as _PyGoExt, build_ex
 from setuptools import Command, find_packages
 from setuptools.command.build_py import build_py as _build_py
 from pkg_resources import working_set, EntryPoint
+from distutils.file_util import copy_file
 from distutils.errors import DistutilsExecError
 from subprocess import Popen, PIPE
 
 import os
 import sys
+from errno import EEXIST
 
 
 # tell cython to resolve `cimport wendelin.*` modules hierarchy starting at top-level.
@@ -172,11 +174,23 @@ def viamake(target, help_text):
 
 # build_ext that
 # - builds via Makefile  (and thus pre-builds ccan)  XXX hacky
+# - integrates built wcfs/wcfs.go exe into wcfs/ python package
 class build_ext(_build_ext):
 
     def run(self):
         runmake('all')
-        return _build_ext.run(self)
+        _build_ext.run(self)
+
+        # copy wcfs/wcfs built from wcfs.go by make into build/lib.linux-x86_64-2.7/...
+        # so that py packaging tools see built wcfs.go as part of wendelin/wcfs package.
+        dst = os.path.join(self.build_lib, 'wendelin/wcfs')
+        try:
+            os.makedirs(dst)
+        except OSError as e:
+            if e.errno != EEXIST:
+                raise
+        copy_file('wcfs/wcfs', dst, verbose=self.verbose, dry_run=self.dry_run)
+
 
 
 
@@ -329,6 +343,9 @@ setup(
                   },
 
     entry_points= {'console_scripts': [
+                        # start wcfs for zurl
+                        'wcfs           = wendelin.wcfs:main',
+
                         # demo to test
                         'demo-zbigarray = wendelin.demo.demo_zbigarray:main',
                       ]

wcfs/.gitignore

+/wcfs
+*.test

wcfs/__init__.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2018-2021  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+
+"""Module wcfs.py provides python gateway for spawning wcfs server.
+"""
+
+from __future__ import print_function, absolute_import
+
+import os, sys
+from os.path import dirname
+
+
+# _wcfs_exe returns path to wcfs executable.
+def _wcfs_exe():
+    return '%s/wcfs' % dirname(__file__)
+
+def main():
+    argv = sys.argv[1:]
+    os.execv(_wcfs_exe(), [_wcfs_exe()] + argv)

wcfs/go.mod

+module lab.nexedi.com/nexedi/wendelin.core/wcfs
+
+go 1.14

wcfs/wcfs.go

+// Copyright (C) 2018-2021  Nexedi SA and Contributors.
+//                          Kirill Smelkov <kirr@nexedi.com>
+//
+// This program is free software: you can Use, Study, Modify and Redistribute
+// it under the terms of the GNU General Public License version 3, or (at your
+// option) any later version, as published by the Free Software Foundation.
+//
+// You can also Link and Combine this program with other software covered by
+// the terms of any of the Free Software licenses or any of the Open Source
+// Initiative approved licenses and Convey the resulting work. Corresponding
+// source of such a combination shall include the source code for all other
+// software used.
+//
+// This program is distributed WITHOUT ANY WARRANTY; without even the implied
+// warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+//
+// See COPYING file for full licensing terms.
+// See https://www.nexedi.com/licensing for rationale and options.
+
+package main
+
+func main() {
+	// XXX stub
+}

wcfs/wcfs_test.py

+# -*- coding: utf-8 -*-
+# Copyright (C) 2018-2021  Nexedi SA and Contributors.
+#                          Kirill Smelkov <kirr@nexedi.com>
+#
+# This program is free software: you can Use, Study, Modify and Redistribute
+# it under the terms of the GNU General Public License version 3, or (at your
+# option) any later version, as published by the Free Software Foundation.
+#
+# You can also Link and Combine this program with other software covered by
+# the terms of any of the Free Software licenses or any of the Open Source
+# Initiative approved licenses and Convey the resulting work. Corresponding
+# source of such a combination shall include the source code for all other
+# software used.
+#
+# This program is distributed WITHOUT ANY WARRANTY; without even the implied
+# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+#
+# See COPYING file for full licensing terms.
+# See https://www.nexedi.com/licensing for rationale and options.
+"""wcfs_test.py tests wcfs filesystem from outside as python client process.
+"""
+
+from __future__ import print_function, absolute_import
+
+from wendelin import wcfs
+
+# TODO