Commit 6f2caaa2 authored by Julien Muchembled's avatar Julien Muchembled

re6stnet: clean-up & backup

See merge request nexedi/slapos!940
parents b4823d48 aad7050a
Pipeline #14711 failed with stage
in 0 seconds
......@@ -8,7 +8,6 @@ LoadModule version_module modules/mod_version.so
LoadModule proxy_module modules/mod_proxy.so
LoadModule proxy_http_module modules/mod_proxy_http.so
LoadModule socache_shmcb_module modules/mod_socache_shmcb.so
LoadModule ssl_module modules/mod_ssl.so
LoadModule mime_module modules/mod_mime.so
#LoadModule dav_module modules/mod_dav.so
#LoadModule dav_fs_module modules/mod_dav_fs.so
......@@ -31,17 +30,6 @@ ErrorLog "{{ error_log }}"
LogFormat "%h %l %u %t \"%r\" %>s %b \"%{Referer}i\" \"%{User-Agent}i\" %D" combined
CustomLog "{{ access_log }}" combined
{% if uri_scheme == 'https' -%}
# SSL Configuration
SSLCertificateFile {{ certificate }}
SSLCertificateKeyFile {{ key }}
SSLRandomSeed startup builtin
SSLRandomSeed connect builtin
SSLProtocol all -SSLv2 -SSLv3
SSLCipherSuite ECDH+AESGCM:DH+AESGCM:ECDH+AES256:DH+AES256:ECDH+AES128:DH+AES:ECDH+3DES:DH+3DES:RSA+AESGCM:RSA+AES:RSA+3DES:HIGH:!aNULL:!MD5
SSLHonorCipherOrder on
{% endif -%}
<Directory />
Options FollowSymLinks
AllowOverride None
......@@ -50,10 +38,5 @@ SSLHonorCipherOrder on
Listen {{ ipv6 }}:{{ apache_port }}
<VirtualHost *:{{ apache_port }}>
{% if uri_scheme == 'https' -%}
SSLEngine On
SSLProxyEngine On
{% endif -%}
ProxyPass / http://{{ re6st_ipv4 }}:{{ re6st_port }}/
</VirtualHost>
......@@ -14,24 +14,16 @@
# not need these here).
[template]
filename = instance.cfg.in
md5sum = 610fc6fd0444d3bab3fca4478572749a
md5sum = 01938aa1683b7994f814fff6d964b9ba
[template-re6stnet]
filename = instance-re6stnet.cfg.in
md5sum = 002f7405f565c82219b0d4b92790ff8c
md5sum = 4e4475c8ef80a3a53aa63a2ae444586d
[template-apache-conf]
filename = apache.conf.in
md5sum = 2ed3c4e9b9d58d2e57cda227bdd454d2
md5sum = 3d55f7c9c4fc7279f06bfe6313a78a4b
[template-re6st-registry-conf]
filename = re6st-registry.conf.in
md5sum = b0f0facfea82a4481f4fb5b0b263e09a
[template-wrapper]
filename = wrapper.in
md5sum = 7633bdfc0e638ff9979f963fbbca8f13
[template-registry-run]
filename = registry-run.in
md5sum = 0bf4f2c03e06b55c6c6cc55fa33e65d6
md5sum = feb4b3318f37414d1bf3d16a03aec93d
......@@ -78,6 +78,11 @@
"description": "Prevent tunnelling accross borders of listed countries",
"type": "string",
"default": ""
},
"backup-repository": {
"title": "Git backup repository",
"description": "URL of Git repository where backups are pushed. Nothing is pushed if empty.",
"type": "string"
}
}
}
{% set bin_directory = parameter_dict['bin-directory'] -%}
{% set python_bin = parameter_dict['python-executable'] -%}
{% set re6st_registry = parameter_dict['re6st-registry'] -%}
{% set re6stnet = parameter_dict['re6stnet'] -%}
{% set publish_dict = {} -%}
{% set part_list = [] -%}
{% set ipv6 = (ipv6_set | list)[0] -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set uri_scheme = slapparameter_dict.get('uri-scheme', 'http') -%}
{% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
[directory]
recipe = slapos.cookbook:mkdirectory
......@@ -19,35 +14,16 @@ log = ${:var}/log
services = ${:etc}/service
script = ${:etc}/run
run = ${:var}/run
ca-dir = ${:etc}/ssl
requests = ${:ca-dir}/requests
private = ${:ca-dir}/private
certs = ${:ca-dir}/certs
newcerts = ${:ca-dir}/newcerts
crl = ${:ca-dir}/crl
re6st = ${:srv}/res6stnet
[re6stnet-dirs]
recipe = slapos.cookbook:mkdirectory
registry = ${directory:re6st}/registry
data = ${directory:srv}/re6stnet
log = ${directory:log}/re6stnet
conf = ${directory:etc}/re6stnet
ssl = ${:conf}/ssl
token = ${:conf}/token
run = ${directory:run}/re6stnet
[certificate-authority]
recipe = slapos.cookbook:certificate_authority
openssl-binary = {{ openssl_bin }}/openssl
ca-dir = ${directory:ca-dir}
requests-directory = ${directory:requests}
wrapper = ${directory:services}/certificate_authority
ca-private = ${directory:private}
ca-certs = ${directory:certs}
ca-newcerts = ${directory:newcerts}
ca-crl = ${directory:crl}
[apache-conf]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-apache-conf'] }}
......@@ -64,56 +40,33 @@ context =
key access_log :access-log
key error_log :error-log
key pid_file :pid-file
raw certificate ${directory:certs}/apache.crt
raw key ${directory:private}/apache.key
raw ipv6 {{ ipv6 }}
raw uri_scheme {{ uri_scheme }}
{% set apache_wrapper = '${directory:services}/httpd' -%}
{% if uri_scheme == 'https' -%}
{% set apache_wrapper = '${directory:bin}/httpd_raw' -%}
{% endif -%}
[apache-httpd]
recipe = slapos.cookbook:wrapper
wrapper-path = {{ apache_wrapper }}
command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
{% if uri_scheme == 'https' %}
[apache-ca]
<= certificate-authority
recipe = slapos.cookbook:certificate_authority.request
executable = ${apache-httpd:wrapper-path}
wrapper = ${directory:bin}/httpd
key-file = ${certificate-authority:ca-private}/apache.key
cert-file = ${certificate-authority:ca-certs}/apache.crt
[{{ section('apache-ca-service') }}]
recipe = slapos.cookbook:wrapper
command-line = ${apache-ca:wrapper}
wrapper-path = ${directory:services}/httpd
hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
{% endif %}
command-line = "{{ parameter_dict['apache-location'] }}/bin/httpd" -f "${apache-conf:rendered}" -DFOREGROUND
[apache-httpd-graceful]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-wrapper'] }}
rendered = ${directory:script}/httpd-graceful
mode = 0700
context =
raw content {{ parameter_dict['apache-location'] }}/bin/httpd -Sf ${apache-conf:rendered}; if [ $? -eq 0 ]; then kill -USR1 $(cat ${apache-conf:pid-file}); fi
raw dash {{ dash_binary }}
template = inline:{{'{{content}}'}}
context = key content :script
script =
#!/bin/sh -e
{{ parameter_dict['apache-location'] }}/bin/httpd -Sf ${apache-conf:rendered}
{{ bin_directory }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
[logrotate-apache]
< = logrotate-entry-base
name = apache
log = ${apache-conf:error-log} ${apache-conf:access-log}
post = test ! -s ${apache-conf:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
post = test ! -s ${apache-conf:pid-file} || {{ bin_directory }}/slapos-kill --pidfile ${apache-conf:pid-file} -s USR1
[re6st-registry-conf-dict]
port = 9201
ipv4 = {{ ipv4 }}
ipv6 = {{ ipv6 }}
db = ${re6stnet-dirs:registry}/registry.db
db = ${re6stnet-dirs:data}/registry.db
ca = ${re6stnet-dirs:ssl}/re6stnet.crt
key = ${re6stnet-dirs:ssl}/re6stnet.key
dh = ${re6stnet-dirs:ssl}/dh.pem
......@@ -135,18 +88,35 @@ same-country = {{ slapparameter_dict.get('same-country', '') }}
[re6st-registry-conf]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-re6st-registry-conf'] }}
rendered = ${directory:etc}/re6st-registry.conf
rendered = ${re6stnet-dirs:conf}/registry.conf
context = section parameter_dict re6st-registry-conf-dict
depends = ${re6st-compat:recipe}
[re6st-compat]
recipe = slapos.recipe.build
update =
import errno, os
res6stnet = self.buildout['directory']['srv'] + '/res6stnet'
try:
os.rename(res6stnet + '/registry', self.buildout['re6stnet-dirs']['data'])
except OSError as e:
if e.errno != errno.ENOENT:
raise
else:
os.rmdir(res6stnet)
[re6st-registry-wrapper]
recipe = slapos.recipe.template:jinja2
template = {{ parameter_dict['template-registry-run'] }}
rendered = ${directory:services}/re6st-registry
pid-file = ${directory:run}/registry.pid
context =
key pid_file :pid-file
raw re6st_command {{ re6st_registry }}
key re6st_conf re6st-registry-conf:rendered
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:services}/re6st-registry
pidfile = ${directory:run}/registry.pid
command-line = {{ bin_directory }}/re6st-registry @${re6st-registry-conf:rendered}
[cron-entry-re6st-backup]
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = re6stnet-backup
time = hourly
command = {{ parameter_dict['re6stnet-backup'] }} ${logrotate-directory:backup}/re6stnet ${re6st-registry-conf-dict:db} {{ slapparameter_dict.get('backup-repository', '') }}
[re6st-registry]
recipe = slapos.cookbook:re6stnet.registry
......@@ -167,7 +137,7 @@ key-file = ${re6st-registry-conf-dict:key}
cert-file = ${re6st-registry-conf-dict:ca}
dh-file = ${re6st-registry-conf-dict:dh}
slave-instance-list = ${slap-parameter:slave_instance_list}
slave-instance-list = {{ slapparameter_dict.get('slave_instance_list', '{}') }}
environment =
PATH={{ openssl_bin }}
......@@ -181,14 +151,14 @@ command-line = "{{ python_bin }}" ${re6st-registry:manager-wrapper}
recipe = slapos.cookbook:cron.d
cron-entries = ${cron:cron-entries}
name = re6stnet-check-token
frequency = */5 * * * *
time = *:0/5
command = {{ python_bin }} ${re6st-registry:manager-wrapper}
[logrotate-entry-re6stnet]
< = logrotate-entry-base
name = re6stnet
log = ${re6st-registry-conf-dict:logfile}
post = test ! -s ${re6st-registry-wrapper:pid-file} || {{ parameter_dict['bin-directory'] }}/slapos-kill --pidfile ${re6st-registry-wrapper:pid-file} -s USR1
post = [ ! -s ${re6st-registry-wrapper:pidfile} ] || {{ bin_directory }}/slapos-kill --pidfile ${re6st-registry-wrapper:pidfile} -s USR1
[port-redirection]
recipe = slapos.recipe.template:jinja2
......@@ -214,7 +184,7 @@ name = apache-re6st-registry.py
config-hostname = ${apache-conf:ipv6}
config-port = ${apache-conf:port}
{% do publish_dict.__setitem__('re6stry-url', uri_scheme ~ '://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
{% do publish_dict.__setitem__('re6stry-url', 'http://[${apache-conf:ipv6}]:${apache-conf:port}') -%}
{% do publish_dict.__setitem__('re6stry-local-url', 'http://${re6st-registry:ipv4}:${re6st-registry:port}/') -%}
{% do publish_dict.__setitem__('slave-amount', '${re6st-registry:slave-amount}') -%}
[publish]
......@@ -230,12 +200,12 @@ extends =
{{ logrotate_cfg }}
parts =
certificate-authority
logrotate-apache
logrotate-entry-re6stnet
re6stnet-manage
cron-entry-logrotate
cron-entry-re6st-manage
cron-entry-re6st-backup
apache-httpd
apache-httpd-graceful
publish
......@@ -244,12 +214,3 @@ parts =
re6st-registry-promise
apache-registry-promise
monitor-base
# Complete parts with sections
{{ part_list | join('\n ') }}
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[slap-parameter]
slave_instance_list = {}
......@@ -16,27 +16,21 @@ recipe = slapos.recipe.template:jinja2
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:filename}
extra-context =
context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key ipv6_set slap-configuration:ipv6
key ipv4_set slap-configuration:ipv4
key slapparameter_dict slap-configuration:configuration
key computer_id slap-configuration:computer
raw logrotate_cfg {{ template_logrotate_base }}
raw dash_binary {{ dash_location }}/bin/dash
raw openssl_bin {{ openssl_location}}/bin
${:extra-context}
[dynamic-template-re6stnet-parameters]
bin-directory = {{ bin_directory }}
python-executable = {{ python_with_eggs }}
re6st-registry = {{ bin_directory }}/re6st-registry
re6stnet = {{ bin_directory }}/re6stnet
template-apache-conf = {{ template_apache_conf }}
template-wrapper = {{ template_wrapper }}
apache-location = {{ apache_location }}
re6stnet-backup = {{ re6stnet_backup }}
template-re6st-registry-conf = {{ template_re6st_registry_conf }}
template-registry-run = {{ template_registry_run }}
[dynamic-template-re6stnet]
< = jinja2-template-base
......@@ -48,7 +42,9 @@ extra-context =
raw monitor2_template_rendered {{ monitor2_template_rendered }}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
recipe = slapos.cookbook:switch-softwaretype
default = ${dynamic-template-re6stnet:rendered}
default = dynamic-template-re6stnet:rendered
registry = ${:default}
# BBB
RootSoftwareInstance = ${:default}
port {{ parameter_dict['port'] }}
4 {{ parameter_dict['ipv4'] }}
authorized-origin {{ parameter_dict['ipv4'] }}
# 6 {{ parameter_dict['ipv6'] }}
6 ''
db {{ parameter_dict['db'] }}
ca {{ parameter_dict['ca'] }}
key {{ parameter_dict['key'] }}
......
#!/bin/bash
echo $$ > {{ pid_file }}
exec {{ re6st_command }} @{{ re6st_conf }}
......@@ -2,13 +2,10 @@
extends =
buildout.hash.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/gzip/buildout.cfg
../../component/openssl/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/apache/buildout.cfg
../../component/pycurl/buildout.cfg
../../component/git/buildout.cfg
../../component/python-cryptography/buildout.cfg
../../stack/slapos.cfg
......@@ -17,10 +14,22 @@ extends =
parts +=
slapos-cookbook
dash
template
[re6st-eggs]
[re6stnet-repository]
recipe = slapos.recipe.build:gitclone
repository = https://lab.nexedi.com/nexedi/re6stnet.git
git-executable = ${git:location}/bin/git
[re6stnet-setup-env]
PATH = ${git:location}/bin:%(PATH)s
[re6stnet-develop]
recipe = zc.recipe.egg:develop
setup = ${re6stnet-repository:location}
environment = re6stnet-setup-env
[re6stnet]
recipe = zc.recipe.egg
interpreter = python-${:_buildout_section_name_}
eggs =
......@@ -28,6 +37,33 @@ eggs =
${python-cryptography:egg}
pyOpenSSL
re6stnet
initialization =
import os
path = os.environ.get('PATH', '')
os.environ['PATH'] = '${git:location}/bin' + (path and ':' + path)
depends = ${re6stnet-develop:recipe}
[re6stnet-backup]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:bin-directory}/re6st-backup
template =
inline:#!/bin/sh -e
PATH={{git_location}}/bin:{{sqlite3_location}}/bin:$PATH
cd "$1" || {
rm -rf "$1.new"
git init --bare "$1.new"
rm "$1.new"/description "$1.new"/hooks/*
mv "$1.new" "$1"
cd "$1"
}
h=`sqlite3 "$2" .dump |git hash-object --stdin -w`
git update-index --add --cacheinfo 0644 "$h" registry.sql
git diff --cached --quiet || GIT_WORK_TREE=$PWD GIT_DIR=$PWD git \
-c gc.auto=100 -c gc.autoDetach=false commit --allow-empty-message -qm ''
[ ! "$3" ] || git push --mirror "$3"
context =
key git_location git:location
key sqlite3_location sqlite3:location
[download-base]
recipe = slapos.recipe.build:download
......@@ -43,17 +79,15 @@ context =
key develop_eggs_directory buildout:develop-eggs-directory
key eggs_directory buildout:eggs-directory
key apache_location apache:location
key dash_location dash:location
key logrotate_location logrotate:location
key openssl_location openssl:location
key re6stnet_backup re6stnet-backup:rendered
key template_apache_conf template-apache-conf:target
key template_re6stnet template-re6stnet:target
key template_re6st_registry_conf template-re6st-registry-conf:target
key template_logrotate_base template-logrotate-base:rendered
key template_wrapper template-wrapper:target
key template_registry_run template-registry-run:target
key monitor2_template_rendered monitor2-template:rendered
raw python_with_eggs ${buildout:bin-directory}/${re6st-eggs:interpreter}
raw python_with_eggs ${buildout:bin-directory}/${re6stnet:interpreter}
[template-re6stnet]
< = download-base
......@@ -64,15 +98,5 @@ context =
[template-re6st-registry-conf]
< = download-base
[template-wrapper]
< = download-base
[template-registry-run]
< = download-base
[versions]
re6stnet = 0.551
# Required by:
# re6stnet==0.533
miniupnpc = 1.9
......@@ -29,7 +29,6 @@ import os
import requests
import json
from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, Re6stnetTestCase = makeModuleSetUpAndTestCaseClass(
......@@ -57,35 +56,3 @@ class TestPortRedirection(Re6stnetTestCase):
'srcPort': 9201,
'destPort': 9201,
}, portredir_config[0])
class ServicesTestCase(Re6stnetTestCase):
@classmethod
def getInstanceParameterDict(cls):
return {'uri-scheme': 'https'}
def test_hashes(self):
hash_files = [
'software_release/buildout.cfg',
]
expected_process_names = [
'httpd-{hash}-on-watch',
]
with self.slap.instance_supervisor_rpc as supervisor:
process_names = [
process['name'] for process in supervisor.getAllProcessInfo()
]
hash_files = [
os.path.join(self.computer_partition_root_path, path)
for path in hash_files
]
for name in expected_process_names:
h = generateHashFromFiles(hash_files)
expected_process_name = name.format(hash=h)
self.assertIn(expected_process_name, process_names)
#!{{ dash }}
{{ content }}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment