Skip to content

slapos
slapos
Commit 4872672a authored by Thomas Gambier's avatar Thomas Gambier 🚴🏼
Browse files
Options

stack/erp5 += WCFS service (technology preview) 

See merge request nexedi/slapos!939
parents 0983f1ca 48b24182
Hide whitespace changes
Inline Side-by-side
Showing with 277 additions and 47 deletions
software/erp5/instance-erp5-input-schema.json
View file @ 4872672a
......@@ -477,6 +477,16 @@
},
"type": "object"
},
"wcfs": {
"description": "Parameters for wendelin.core filesystem",
"properties": {
"enable": {
"description": "Whether to enable WCFS filesystem and use it to access ZBigArray/ZBigFile data. In WCFS mode wendelin.core clients (Zope/ERP5 processes) share in-RAM cache for in-ZODB data without duplicating it for every client. This cache sharing does not affect correctness as isolation property is continued to be provided to every client.",
"default": false,
"type": "boolean"
}
}
},
"wendelin-core-zblk-fmt": {
"description": "In wendelin.core there are 2 formats for storing data, so called ZBlk0 and ZBlk1. See https://lab.nexedi.com/nexedi/wendelin.core/blob/2e5e1d3d/bigfile/file_zodb.py#L19 for more details.",
"default": "",
......
software/erp5/test/test/test_wcfs.py 0 → 100644
View file @ 4872672a
# Copyright (C) 2021 Nexedi SA and Contributors.
#
# This program is free software: you can Use, Study, Modify and Redistribute
# it under the terms of the GNU General Public License version 3, or (at your
# option) any later version, as published by the Free Software Foundation.
#
# You can also Link and Combine this program with other software covered by
# the terms of any of the Free Software licenses or any of the Open Source
# Initiative approved licenses and Convey the resulting work. Corresponding
# source of such a combination shall include the source code for all other
# software used.
#
# This program is distributed WITHOUT ANY WARRANTY; without even the implied
# warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
#
# See COPYING file for full licensing terms.
# See https://www.nexedi.com/licensing for rationale and options.
import json
import os.path
import unittest
from slapos.grid.utils import md5digest
from . import ERP5InstanceTestCase, setUpModule as _setUpModule
from .test_erp5 import TestPublishedURLIsReachableMixin
# skip tests when software release is built with wendelin.core 1.
def setUpModule():
_setUpModule()
cls = ERP5InstanceTestCase
if not os.path.exists(
os.path.join(
cls.slap.software_directory,
md5digest(cls.getSoftwareURL()),
'bin', 'wcfs')):
raise unittest.SkipTest("built with wendelin.core 1")
class TestWCFS(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test Wendelin Core File System
"""
__partition_reference__ = 'wcfs'
@classmethod
def getInstanceParameterDict(cls):
return {'_': json.dumps({'wcfs': {'enable': True}})}
def test_wcfs_accessible(self):
"""Verify that wcfs filesystem is basically accessible.
- we can read .wcfs/zurl
- its content is equal to published `serving-zurl`
"""
zurl = json.loads(
self.getComputerPartition('wcfs').getConnectionParameter('_')
)['serving-zurl']
mntpt = lookupMount(zurl)
zurl_ = readfile("%s/.wcfs/zurl" % mntpt)
self.assertEqual(zurl_, zurl)
# lookupMount returns /proc/mount entry for wcfs mounted to serve zurl.
def lookupMount(zurl):
for line in readfile('/proc/mounts').splitlines():
# <zurl> <mountpoint> fuse.wcfs ...
zurl_, mntpt, typ, _ = line.split(None, 3)
if typ != 'fuse.wcfs':
continue
if zurl_ == zurl:
return mntpt
raise KeyError("lookup mount %s: no /proc/mounts entry" % zurl)
# readfile returns content of file @path.
def readfile(path):
with open(path, 'r') as f:
return f.read()
software/slapos-master/buildout.hash.cfg
View file @ 4872672a
......@@ -14,7 +14,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = f078d72db690805eab90a4577066b920
md5sum = eba73f868a5bbfb942323eadcd88cb16
[template-balancer]
filename = instance-balancer.cfg.in
......
software/slapos-master/instance-erp5.cfg.in
View file @ 4872672a
......@@ -9,6 +9,8 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
......@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
{% endif -%}
{% endfor -%}
[request-zodb-base]
config-zodb-dict = {{ dumps(zodb_dict) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
{% set zope_address_list_id_dict = {} -%}
{% if zope_partition_dict -%}
[request-zope-base]
<= request-common
request-zodb-base
return =
zope-address-list
hosts-dict
......@@ -233,20 +250,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-zodb-dict = {{ dumps(zodb_dict) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
config-wcfs_enable = {{ wcfs_enable }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......@@ -348,6 +354,17 @@ config-url = ${request-jupyter:connection-url}
{% endif -%}
{%- endif %}
{% if wcfs_enable -%}
{# request WCFS connected to ZODB -#}
{% do root_common.section('request-wcfs') -%}
{{ request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
[request-wcfs]
<= request-common
request-zodb-base
{%- endif %}
[directory]
recipe = slapos.cookbook:mkdirectory
{% if slapparameter_dict.get('shared-certificate-authority-path', '') -%}
......
stack/erp5/buildout.cfg
View file @ 4872672a
......@@ -185,6 +185,7 @@ context =
key instance_common_cfg instance-common:rendered
key jsl_location jsl:location
key jupyter_enable_default erp5-defaults:jupyter-enable-default
key wcfs_enable_default erp5-defaults:wcfs-enable-default
key kumo_location kumo:location
key local_bt5_repository local-bt5-repository:list
key logrotate_location logrotate:location
......@@ -222,7 +223,9 @@ context =
key template_postfix_aliases template-postfix-aliases:target
key template_postfix_main_cf template-postfix-main-cf:target
key template_postfix_master_cf template-postfix-master-cf:target
key instance_wcfs_cfg_in instance-wcfs.cfg.in:target
key template_zeo template-zeo:target
key template_zodb_base template-zodb-base:target
key template_zope template-zope:target
key template_zope_conf template-zope-conf:target
key template_fonts_conf template-fonts-conf:output
......@@ -237,6 +240,9 @@ context =
[template-zeo]
<= download-base
[template-zodb-base]
<= download-base
[template-zope]
<= download-base
link-binary =
......@@ -281,6 +287,9 @@ fontconfig-includes =
[template-rsyslogd-cfg]
<= download-base
[instance-wcfs.cfg.in]
<= download-base
[erp5-bin]
<= erp5
repository = https://lab.nexedi.com/nexedi/erp5-bin.git
......@@ -320,6 +329,8 @@ repository_id_list = erp5 erp5-bin erp5-doc
cloudooo-connection-url = https://cloudooo.erp5.net/
# Jupyter is by default disabled in ERP5
jupyter-enable-default = false
# WCFS is by default disabled in ERP5
wcfs-enable-default = false
[erp5]
recipe = slapos.recipe.build:gitclone
......@@ -558,6 +569,7 @@ scripts =
runzeo
tidstoraged
tidstorage_repozo
wcfs
web_checker_utility
extra-paths =
......
stack/erp5/buildout.hash.cfg
View file @ 4872672a
......@@ -70,19 +70,23 @@ md5sum = 7a14019abf48ca100eb94d9add20f5ae
[template]
filename = instance.cfg.in
md5sum = b5ac16fdeed8863e465e955ba6d1e12a
md5sum = 8bd7f89b7c1e453ecc952659a01347e6
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = ac155f28a096747fc267f32a1cec46e1
md5sum = 79f092c54998918423257f4b676425ea
[template-zeo]
filename = instance-zeo.cfg.in
md5sum = 0648e38bd5d3a15bb9f93264932740b9
[template-zodb-base]
filename = instance-zodb-base.cfg.in
md5sum = bc821f9f9696953b10a03ad7b59a1936
[template-zope]
filename = instance-zope.cfg.in
md5sum = 9fa66b93fbf6a40aa8136c651ad9f539
md5sum = e495f7225a49f61c501ccc496a976d63
[template-balancer]
filename = instance-balancer.cfg.in
......@@ -95,3 +99,7 @@ md5sum = 3f4f7e49c504cbf610fc5dc462713dfc
[template-rsyslogd-cfg]
filename = rsyslogd.cfg.in
md5sum = 5cf0316fdd17a940031e4083bbededd8
[instance-wcfs.cfg.in]
filename = instance-wcfs.cfg.in
md5sum = 945e8e4552a6bdf228b9609567b09399
stack/erp5/instance-erp5.cfg.in
View file @ 4872672a
......@@ -9,6 +9,8 @@
{% set jupyter_dict = slapparameter_dict.get('jupyter', {}) -%}
{% set has_jupyter = jupyter_dict.get('enable', jupyter_enable_default.lower() in ('true', 'yes')) -%}
{% set jupyter_zope_family = jupyter_dict.get('zope-family', '') -%}
{% set wcfs_dict = slapparameter_dict.get('wcfs', {}) -%}
{% set wcfs_enable = wcfs_dict.get('enable', wcfs_enable_default.lower() in ('true', 'yes')) -%}
{% set test_runner_enabled = slapparameter_dict.get('test-runner', {}).get('enabled', True) -%}
{% set test_runner_node_count = slapparameter_dict.get('test-runner', {}).get('node-count', 3) -%}
{% set test_runner_extra_database_count = slapparameter_dict.get('test-runner', {}).get('extra-database-count', 3) -%}
......@@ -189,12 +191,27 @@ connection-url = smtp://127.0.0.2:0/
{% endif -%}
{% endfor -%}
[request-zodb-base]
config-zodb-dict = {{ dumps(zodb_dict) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
{% set zope_address_list_id_dict = {} -%}
{% if zope_partition_dict -%}
[request-zope-base]
<= request-common
request-zodb-base
return =
zope-address-list
hosts-dict
......@@ -238,20 +255,9 @@ config-timezone = {{ dumps(slapparameter_dict.get('timezone', 'UTC')) }}
config-cloudooo-retry-count = {{ slapparameter_dict.get('cloudooo-retry-count', 2) }}
config-wendelin-core-zblk-fmt = {{ dumps(slapparameter_dict.get('wendelin-core-zblk-fmt', '')) }}
config-wsgi = {{ dumps(slapparameter_dict.get('wsgi', True)) }}
config-zodb-dict = {{ dumps(zodb_dict) }}
config-test-runner-enabled = {{ dumps(test_runner_enabled) }}
config-test-runner-node-count = {{ dumps(test_runner_node_count) }}
{% for server_type, server_dict in storage_dict.iteritems() -%}
{% if server_type == 'neo' -%}
config-neo-cluster = ${publish-early:neo-cluster}
config-neo-name = {{ server_dict.keys()[0] }}
config-neo-masters = ${publish-early:neo-masters}
{% else -%}
config-zodb-zeo = ${request-zodb:connection-storage-dict}
config-tidstorage-ip = ${request-zodb:connection-tidstorage-ip}
config-tidstorage-port = ${request-zodb:connection-tidstorage-port}
{% endif -%}
{% endfor -%}
config-wcfs_enable = {{ wcfs_enable }}
software-type = zope
{% set global_publisher_timeout = slapparameter_dict.get('publisher-timeout') -%}
......@@ -353,6 +359,17 @@ config-url = ${request-jupyter:connection-url}
{% endif -%}
{%- endif %}
{% if wcfs_enable -%}
{# request WCFS connected to ZODB -#}
{% do root_common.section('request-wcfs') -%}
{{ request('wcfs', 'wcfs', 'wcfs', {}, {}) }}
[request-wcfs]
<= request-common
request-zodb-base
{%- endif %}
{% set balancer_ret_dict = {'monitor-base-url': False} -%}
{% for family in zope_family_dict -%}
{% do balancer_ret_dict.__setitem__(family, False) -%}
......
stack/erp5/instance-wcfs.cfg.in 0 → 100644
View file @ 4872672a
{# instance that runs WCFS service associated with ZODB storage #}
{% from "instance_zodb_base" import zodb_dict with context %}
{# build zurl to connect to configured ZODB #}
{% if len(zodb_dict) != 1 -%}
{% do assert(False, ("WCFS supports only single ZODB storage", zodb_dict)) -%}
{% endif -%}
{% set db_name, zodb = zodb_dict.popitem() -%}
{% set z = zodb['storage-dict'] -%}
{% if zodb['type'] == 'zeo' -%}
{% set zurl = ('zeo://%s?storage=%s' % (z['server'], z['storage'])) -%}
{% elif zodb['type'] == 'neo' -%}
{% set zurl = ('neo://%s@%s' % (z.pop('name'), z.pop('master_nodes'))) -%}
{% set argv = [] -%}
{% set i = 0 -%}
{% for k,v in z|dictsort -%}
{% do argv.append('%s=%s' % (k,v)) -%}
{% endfor -%}
{% if len(argv) > 0 -%}
{% set zurl = zurl + '&' + '?'.join(argv) -%}
{% endif -%}
{% else -%}
{% do assert(False, ("unsupported ZODB type", zodb)) -%}
{% endif -%}
[buildout]
extends = {{ template_monitor }}
parts +=
wcfs
wcfs-promise
publish
[directory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
log = ${:var}/log
run = ${:var}/run
services = ${:etc}/run
service-on-watch = ${:etc}/service
srv = ${buildout:directory}/srv
tmp = ${buildout:directory}/tmp
var = ${buildout:directory}/var
[wcfs]
recipe = slapos.cookbook:wrapper
command-line = {{ bin_directory }}/wcfs serve -log_dir=${directory:log} {{ zurl }}
wrapper-path = ${directory:service-on-watch}/wcfs
[wcfs-promise]
<= monitor-promise-base
module = check_command_execute
name = ${:_buildout_section_name_}.py
config-command = {{ bin_directory }}/wcfs status {{ zurl }}
[publish]
recipe = slapos.cookbook:publish.serialised
serving-zurl = {{ zurl }}
stack/erp5/instance-zodb-base.cfg.in 0 → 100644
View file @ 4872672a
{# base for instances that need to access ZODB storage #}
{# provides zodb_dict #}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
{% for name, zodb in zodb_dict.iteritems() -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{% if zodb['type'] == 'zeo' -%}
{% do storage_dict.update(zeo_dict.get(name, ())) -%}
{% else -%}
{% if name == slapparameter_dict.get('neo-name') -%}
{% do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
name=slapparameter_dict['neo-cluster']) -%}
{% endif -%}
{{ assert(storage_dict['master_nodes'], name) }}
{% if storage_dict.pop('ssl', 1) -%}
{% do storage_dict.update(ca='~/etc/ca.crt',
cert='~/etc/neo.crt',
key='~/etc/neo.key') -%}
{% endif -%}
{% endif -%}
{% endfor -%}
stack/erp5/instance-zope.cfg.in
View file @ 4872672a
{% from "instance_zodb_base" import zodb_dict with context %}
{% set wsgi = slapparameter_dict['wsgi'] -%}
{% set webdav = slapparameter_dict['webdav'] -%}
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
{% set site_id = slapparameter_dict['site-id'] -%}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set instance_index_list = range(slapparameter_dict['instance-count']) -%}
{% set node_id_base = slapparameter_dict['name'] -%}
{% set selenium_server_configuration_dict = slapparameter_dict.get('selenium-server-configuration-dict', None) -%}
......@@ -63,6 +63,12 @@ environment +=
FONTCONFIG_FILE=${fontconfig-conf:rendered}
{% if slapparameter_dict.get('wendelin-core-zblk-fmt') %}
WENDELIN_CORE_ZBLK_FMT={{ slapparameter_dict['wendelin-core-zblk-fmt'] }}
{% endif %}
WENDELIN_CORE_WCFS_AUTOSTART=no
{% if slapparameter_dict['wcfs_enable'] %}
WENDELIN_CORE_VIRTMEM=r:wcfs+w:uvmm
{% else %}
WENDELIN_CORE_VIRTMEM=rw:uvmm
{% endif %}
${:environment-extra}
......@@ -257,24 +263,6 @@ cloudooo-url = {{ (cloudooo if cloudooo.port == None else
cloudooo._replace(netloc='erp5-cloudooo:%s' % cloudooo.port)).geturl() }}
{% endif -%}
{% set zeo_dict = slapparameter_dict.get('zodb-zeo', {}) -%}
{% for name, zodb in zodb_dict.iteritems() -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
{% if zodb['type'] == 'zeo' -%}
{% do storage_dict.update(zeo_dict.get(name, ())) -%}
{% else -%}
{% if name == slapparameter_dict.get('neo-name') -%}
{% do storage_dict.update(master_nodes=slapparameter_dict['neo-masters'],
name=slapparameter_dict['neo-cluster']) -%}
{% endif -%}
{{ assert(storage_dict['master_nodes'], name) }}
{% if storage_dict.pop('ssl', 1) -%}
{% do storage_dict.update(ca='~/etc/ca.crt',
cert='~/etc/neo.crt',
key='~/etc/neo.key') -%}
{% endif -%}
{% endif -%}
{% endfor -%}
developer-list = {{ dumps(slapparameter_dict['developer-list']) }}
publisher-timeout = {{ dumps(slapparameter_dict['publisher-timeout']) }}
activity-timeout = {{ dumps(slapparameter_dict['activity-timeout']) }}
......
stack/erp5/instance.cfg.in
View file @ 4872672a
......@@ -31,11 +31,13 @@ extra-context =
[dynamic-template-erp5-parameters]
default-cloudooo-url = {{ dumps(default_cloudooo_url) }}
jupyter-enable-default = {{ jupyter_enable_default }}
wcfs-enable-default = {{ wcfs_enable_default }}
local-bt5-repository = {{ ' '.join(local_bt5_repository.split()) }}
[context]
root-common = {{ root_common }}
caucase-jinja2-library = {{ caucase_jinja2_library }}
template-zodb-base = {{ template_zodb_base }}
[dynamic-template-erp5]
<= jinja2-template-base
......@@ -44,6 +46,7 @@ filename = instance-erp5.cfg
extra-context =
key default_cloudooo_url dynamic-template-erp5-parameters:default-cloudooo-url
key jupyter_enable_default dynamic-template-erp5-parameters:jupyter-enable-default
key wcfs_enable_default dynamic-template-erp5-parameters:wcfs-enable-default
key local_bt5_repository dynamic-template-erp5-parameters:local-bt5-repository
key openssl_location :openssl-location
import re re
......@@ -125,6 +128,8 @@ extra-context =
import urlparse urlparse
import hashlib hashlib
import itertools itertools
import-list =
file instance_zodb_base context:template-zodb-base
[dynamic-template-kumofs-parameters]
<= default-dynamic-template-parameters
......@@ -190,6 +195,15 @@ mode = 644
[dynamic-template-jupyter]
rendered = {{ template_jupyter_cfg }}
[dynamic-template-wcfs]
<= jinja2-template-base
template = {{ instance_wcfs_cfg_in }}
filename = instance_wcfs.cfg
extra-context =
section parameter_dict dynamic-template-zope-parameters
import-list =
file instance_zodb_base context:template-zodb-base
[switch-softwaretype]
recipe = slapos.cookbook:switch-softwaretype
override = {{ dumps(override_switch_softwaretype |default) }}
......@@ -208,6 +222,7 @@ zodb-zeo = dynamic-template-zeo:rendered
zodb-neo = neo:rendered
zope = dynamic-template-zope:rendered
jupyter = dynamic-template-jupyter:rendered
wcfs = dynamic-template-wcfs:rendered
# Keep cloudooo backward compatibility
cloudooo = dynamic-template-legacy:output
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7