stack/erp5 += WCFS service (draft)

This complements 7f877621 (Move wendelin.core from Wendelin to ERP5) and
introduces new WCFS service into ERP5 software-release to support
upcoming wendelin.core 2 - because wendelin.core 2 will introduce new
filesystem server component that serves in-ZODB arrays data as OS files
for Zopes to memory-map.

- WCFS is by default disabled in ERP5 - we do not want to load every
  ERP5 user with it in the early days of wendelin.core 2. WCFS service
  will also not work when ERP5 is built with wendelin.core 1, so keeping
  it disabled by default is reasonable for now.

- There is a duplication for instance-erp5.cfg.in in between stack/erp5/
  and software/slapos-master/ . It was negotiated with VIFIB developers
  to go this way instead of first deduplicating instance-erp5.cfg.in
  copying that was started in 87d13789 (slapos-master: Copy files from
  erp5 stack for local changes).

To enable wcfs one can either pass `wcfs` configuration parameter, or
add the following to a software-release that uses stack/erp5:

  [erp5-defaults]
  wcfs-enable-default = true

/cc @rafael, @tomo

software/slapos-master/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = f078d72db690805eab90a4577066b920
+md5sum = eba73f868a5bbfb942323eadcd88cb16
 
 [template-balancer]
 filename = instance-balancer.cfg.in

software/slapos-master/instance-erp5.cfg.in

@@ -9,6 +9,8 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
+{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
 {% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
 {% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
 {% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
 {%   endif -%}
 {% endfor -%}
 
+[request-zodb-base]
+config-zodb-dict = {{ dumps(zodb_dict) }}
+{% for server_type, server_dict in storage_dict.iteritems() -%}
+{%   if server_type == 'neo' -%}
+config-neo-cluster = ${publish-early:neo-cluster}
+config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-masters = ${publish-early:neo-masters}
+{%   else -%}
+config-zodb-zeo = ${request-zodb:connection-storage-dict}
+config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
+config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
+{%   endif -%}
+{% endfor -%}
+
 
 {% set zope_address_list_id_dict = {} -%}
 {% if zope_partition_dict -%}
 
 [request-zope-base]
 <= request-common
+   request-zodb-base
 return =
   zope-address-list
   hosts-dict
@@ -233,20 +250,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
 config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
 config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
-config-zodb-dict = {{ dumps(zodb_dict) }}
 config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
 config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
-{%   if server_type == 'neo' -%}
-config-neo-cluster = ${publish-early:neo-cluster}
-config-neo-name = {{ server_dict.keys()[0] }}
-config-neo-masters = ${publish-early:neo-masters}
-{%   else -%}
-config-zodb-zeo = ${request-zodb:connection-storage-dict}
-config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
-config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-{%   endif -%}
-{% endfor -%}
+config-wcfs_enable = {{ wcfs_enable }}
 software-type = zope
 
 {% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
@@ -348,6 +354,17 @@ config-url = ${request-jupyter:connection-url}
 {%   endif -%}
 {%- endif %}
 
+{% if wcfs_enable -%}
+{# request WCFS connected to ZODB -#}
+{%   do root_common.section('request-wcfs') -%}
+{{   request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
+
+[request-wcfs]
+<= request-common
+   request-zodb-base
+
+{%- endif %}
+
 [directory]
 recipe = slapos.cookbook:mkdirectory
 {% if slapparameter_dict.get('shared-certificate-authority-path', '') -%}

stack/erp5/buildout.cfg

@@ -187,6 +187,7 @@ context =
     key instance_common_cfg instance-common:rendered
     key jsl_location jsl:location
     key jupyter_enable_default erp5-defaults:jupyter-enable-default
+    key wcfs_enable_default erp5-defaults:wcfs-enable-default
     key kumo_location kumo:location
     key local_bt5_repository local-bt5-repository:list
     key logrotate_location logrotate:location
@@ -224,7 +225,9 @@ context =
     key template_postfix_aliases template-postfix-aliases:target
     key template_postfix_main_cf template-postfix-main-cf:target
     key template_postfix_master_cf template-postfix-master-cf:target
+    key instance_wcfs_cfg_in instance-wcfs.cfg.in:target
     key template_zeo template-zeo:target
+    key template_zodb_base template-zodb-base:target
     key template_zope template-zope:target
     key template_zope_conf template-zope-conf:target
     key template_fonts_conf template-fonts-conf:output
@@ -239,6 +242,9 @@ context =
 [template-zeo]
 <= download-base
 
+[template-zodb-base]
+<= download-base
+
 [template-zope]
 <= download-base
 link-binary =
@@ -282,6 +288,9 @@ fontconfig-includes =
 [template-rsyslogd-cfg]
 <= download-base
 
+[instance-wcfs.cfg.in]
+<= download-base
+
 [erp5-bin]
 <= erp5
 repository = https://lab.nexedi.com/nexedi/erp5-bin.git
@@ -321,6 +330,8 @@ repository_id_list = erp5 erp5-bin erp5-doc
 cloudooo-connection-url = https://cloudooo.erp5.net/
 # Jupyter is by default disabled in ERP5
 jupyter-enable-default = false
+# WCFS is by default disabled in ERP5
+wcfs-enable-default = false
 
 [erp5]
 recipe = slapos.recipe.build:gitclone
@@ -559,6 +570,7 @@ scripts =
   runzeo
   tidstoraged
   tidstorage_repozo
+  wcfs
   web_checker_utility
 
 extra-paths =

stack/erp5/buildout.hash.cfg

@@ -70,19 +70,23 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
 
 [template]
 filename = instance.cfg.in
-md5sum = b5ac16fdeed8863e465e955ba6d1e12a
+md5sum = 8bd7f89b7c1e453ecc952659a01347e6
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = ac155f28a096747fc267f32a1cec46e1
+md5sum = 79f092c54998918423257f4b676425ea
 
 [template-zeo]
 filename = instance-zeo.cfg.in
 md5sum = 0648e38bd5d3a15bb9f93264932740b9
 
+[template-zodb-base]
+filename = instance-zodb-base.cfg.in
+md5sum = bc821f9f9696953b10a03ad7b59a1936
+
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 9fa66b93fbf6a40aa8136c651ad9f539
+md5sum = e495f7225a49f61c501ccc496a976d63
 
 [template-balancer]
 filename = instance-balancer.cfg.in
@@ -95,3 +99,7 @@ md5sum = 452c502fabd5a6066c9dee533dfb1c77
 [template-rsyslogd-cfg]
 filename = rsyslogd.cfg.in
 md5sum = 7030e42b50e03f24e036b7785bd6159f
+
+[instance-wcfs.cfg.in]
+filename = instance-wcfs.cfg.in
+md5sum = 596b2231df29b2acdd03d34d5dc86def

stack/erp5/instance-erp5.cfg.in

@@ -9,6 +9,8 @@
 {% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
 {% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
 {% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
+{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
+{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
 {% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
 {% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
 {% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
 {%   endif -%}
 {% endfor -%}
 
+[request-zodb-base]
+config-zodb-dict = {{ dumps(zodb_dict) }}
+{% for server_type, server_dict in storage_dict.iteritems() -%}
+{%   if server_type == 'neo' -%}
+config-neo-cluster = ${publish-early:neo-cluster}
+config-neo-name = {{ server_dict.keys()[0] }}
+config-neo-masters = ${publish-early:neo-masters}
+{%   else -%}
+config-zodb-zeo = ${request-zodb:connection-storage-dict}
+config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
+config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
+{%   endif -%}
+{% endfor -%}
+
 
 {% set zope_address_list_id_dict = {} -%}
 {% if zope_partition_dict -%}
 
 [request-zope-base]
 <= request-common
+   request-zodb-base
 return =
   zope-address-list
   hosts-dict
@@ -238,20 +255,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
 config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
 config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
 config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
-config-zodb-dict = {{ dumps(zodb_dict) }}
 config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
 config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
-{% for server_type, server_dict in storage_dict.iteritems() -%}
-{%   if server_type == 'neo' -%}
-config-neo-cluster = ${publish-early:neo-cluster}
-config-neo-name = {{ server_dict.keys()[0] }}
-config-neo-masters = ${publish-early:neo-masters}
-{%   else -%}
-config-zodb-zeo = ${request-zodb:connection-storage-dict}
-config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
-config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
-{%   endif -%}
-{% endfor -%}
+config-wcfs_enable = {{ wcfs_enable }}
 software-type = zope
 
 {% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
@@ -353,6 +359,17 @@ config-url = ${request-jupyter:connection-url}
 {%   endif -%}
 {%- endif %}
 
+{% if wcfs_enable -%}
+{# request WCFS connected to ZODB -#}
+{%   do root_common.section('request-wcfs') -%}
+{{   request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
+
+[request-wcfs]
+<= request-common
+   request-zodb-base
+
+{%- endif %}
+
 {% set balancer_ret_dict = {'monitor-base-url': False} -%}
 {% for family in zope_family_dict -%}
 {%   do balancer_ret_dict.__setitem__(family, False) -%}

stack/erp5/instance-wcfs.cfg.in

+{# instance that runs WCFS service associated with ZODB storage  #}
+{% from "instance_zodb_base" import zodb_dict  with context %}
+
+
+{# build zurl to connect to configured ZODB #}
+{% if len(zodb_dict) != 1 -%}
+{%   do assert(False, ("WCFS supports only single ZODB storage", zodb_dict)) -%}
+{% endif -%}
+
+{% set db_name, zodb = zodb_dict.popitem() -%}
+{% set z = zodb['storage-dict'] -%}
+{% if zodb['type'] == 'zeo' -%}
+{%   set zurl = ('zeo://%s?storage=%s' % (z['server'], z['storage'])) -%}
+{% elif zodb['type'] == 'neo' -%}
+{%   set zurl = ('neo://%s@%s' % (z.pop('name'), z.pop('master_nodes'))) -%}
+{%   set argv = [] -%}
+{%   set i = 0 -%}
+{%   for k,v in z|dictsort -%}
+{%     do argv.append('%s=%s' % (k,v)) -%}
+{%   endfor -%}
+{%   if len(argv) > 0 -%}
+{%     set zurl = zurl + '&' + '?'.join(argv) -%}
+{%   endif -%}
+{% else -%}
+{%   do assert(False, ("unsupported ZODB type", zodb)) -%}
+{% endif -%}
+
+
+[buildout]
+extends = {{ template_monitor }}
+parts +=
+  wcfs
+  wcfs-promise
+  publish
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+log = ${:var}/log
+run = ${:var}/run
+services = ${:etc}/run
+service-on-watch = ${:etc}/service
+srv = ${buildout:directory}/srv
+tmp = ${buildout:directory}/tmp
+var = ${buildout:directory}/var
+
+[wcfs]
+recipe = slapos.cookbook:wrapper
+command-line = {{ bin_directory }}/wcfs serve -log_dir=${directory:log} {{ zurl }}
+wrapper-path = ${directory:service-on-watch}/wcfs
+
+[wcfs-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = ${:_buildout_section_name_}.py
+config-command = {{ bin_directory }}/wcfs status {{ zurl }}
+
+[publish]
+recipe = slapos.cookbook:publish
+serving-zurl = {{ zurl }}

stack/erp5/instance-zodb-base.cfg.in

+{# base for instances that need to access ZODB storage #}
+{# provides zodb_dict #}
+
+{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
+
+{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
+{% for name, zodb in zodb_dict.iteritems() -%}
+{%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
+{%   if zodb['type'] == 'zeo' -%}
+{%     do storage_dict.update(zeo_dict.get(name, ())) -%}
+{%   else -%}
+{%     if name == slapparameter_dict.get('neo-name') -%}
+{%       do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
+                                name=slapparameter_dict['neo-cluster']) -%}
+{%     endif -%}
+{{     assert(storage_dict['master_nodes'], name) }}
+{%     if storage_dict.pop('ssl', 1) -%}
+{%       do storage_dict.update(ca='~/etc/ca.crt',
+                                cert='~/etc/neo.crt',
+                                key='~/etc/neo.key') -%}
+{%     endif -%}
+{%   endif -%}
+{% endfor -%}

stack/erp5/instance-zope.cfg.in

+{% from "instance_zodb_base" import zodb_dict  with context %}
 {% set wsgi = slapparameter_dict['wsgi'] -%}
 {% set webdav = slapparameter_dict['webdav'] -%}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 {% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
 {% set site_id = slapparameter_dict['site-id'] -%}
-{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
 {% set instance_index_list = range(slapparameter_dict['instance-count']) -%}
 {% set node_id_base = slapparameter_dict['name'] -%}
 {% set selenium_server_configuration_dict = slapparameter_dict.get('selenium-server-configuration-dict', None) -%}
@@ -63,6 +63,12 @@ environment +=
   FONTCONFIG_FILE=${fontconfig-conf:rendered}
 {% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
   WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
+{% endif %}
+  WENDELIN_CORE_WCFS_AUTOSTART=no
+{% if slapparameter_dict['wcfs_enable'] %}
+  WENDELIN_CORE_VIRTMEM=r:wcfs+w:uvmm
+{% else %}
+  WENDELIN_CORE_VIRTMEM=rw:uvmm
 {% endif %}
   ${:environment-extra}
 
@@ -257,24 +263,6 @@ cloudooo-url = {{ (cloudooo if cloudooo.port == None else
   cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
 
 {% endif -%}
-{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
-{% for name, zodb in zodb_dict.iteritems() -%}
-{%   set storage_dict = zodb.setdefault('storage-dict', {}) -%}
-{%   if zodb['type'] == 'zeo' -%}
-{%     do storage_dict.update(zeo_dict.get(name, ())) -%}
-{%   else -%}
-{%     if name == slapparameter_dict.get('neo-name') -%}
-{%       do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
-                                name=slapparameter_dict['neo-cluster']) -%}
-{%     endif -%}
-{{     assert(storage_dict['master_nodes'], name) }}
-{%     if storage_dict.pop('ssl', 1) -%}
-{%       do storage_dict.update(ca='~/etc/ca.crt',
-                                cert='~/etc/neo.crt',
-                                key='~/etc/neo.key') -%}
-{%     endif -%}
-{%   endif -%}
-{% endfor -%}
 developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
 publisher-timeout = {{ dumps(slapparameter_dict['publisher-timeout']) }}
 activity-timeout = {{ dumps(slapparameter_dict['activity-timeout']) }}

stack/erp5/instance.cfg.in

@@ -31,11 +31,13 @@ extra-context =
 [dynamic-template-erp5-parameters]
 default-cloudooo-url =  {{ dumps(default_cloudooo_url) }}
 jupyter-enable-default = {{ jupyter_enable_default }}
+wcfs-enable-default = {{ wcfs_enable_default }}
 local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
 
 [context]
 root-common = {{ root_common }}
 caucase-jinja2-library = {{ caucase_jinja2_library }}
+template-zodb-base = {{ template_zodb_base }}
 
 [dynamic-template-erp5]
 <= jinja2-template-base
@@ -44,6 +46,7 @@ filename = instance-erp5.cfg
 extra-context =
     key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
     key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
+    key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
     key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
     key openssl_location :openssl-location
     import re re
@@ -125,6 +128,8 @@ extra-context =
     import urlparse urlparse
     import hashlib hashlib
     import itertools itertools
+import-list =
+    file instance_zodb_base context:template-zodb-base
 
 [dynamic-template-kumofs-parameters]
 <= default-dynamic-template-parameters
@@ -190,6 +195,15 @@ mode = 644
 [dynamic-template-jupyter]
 rendered = {{ template_jupyter_cfg }}
 
+[dynamic-template-wcfs]
+<= jinja2-template-base
+template = {{ instance_wcfs_cfg_in }}
+filename = instance_wcfs.cfg
+extra-context =
+    section parameter_dict dynamic-template-zope-parameters
+import-list =
+    file instance_zodb_base context:template-zodb-base
+
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 override = {{ dumps(override_switch_softwaretype |default) }}
@@ -208,6 +222,7 @@ zodb-zeo = dynamic-template-zeo:rendered
 zodb-neo = neo:rendered
 zope = dynamic-template-zope:rendered
 jupyter = dynamic-template-jupyter:rendered
+wcfs = dynamic-template-wcfs:rendered
 
 # Keep cloudooo backward compatibility
 cloudooo = dynamic-template-legacy:output