Commit ca54ed9d authored by Jérome Perrin's avatar Jérome Perrin

software/restic-rest-server: new software

https://github.com/restic/rest-server
parent db30e87b
# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
# The only allowed lines here are (regexes):
# - "^#" comments, copied verbatim
# - "^[" section beginings, copied verbatim
# - lines containing an "=" sign which must fit in the following categorie.
# - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
# Copied verbatim.
# - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
# by the re-generation script.
# Re-generated.
# - other lines are copied verbatim
# Substitution (${...:...}), extension ([buildout] extends = ...) and
# section inheritance (< = ...) are NOT supported (but you should really
# not need these here).
[instance.cfg.in]
filename = instance.cfg.in
md5sum = fee2097f3d12fd4bcfbc2698ecf49afc
{% import "caucase" as caucase with context %}
[buildout]
parts =
promises
publish-connection-parameter
extends =
{{ template_monitor }}
eggs-directory = {{ buildout['eggs-directory'] }}
develop-eggs-directory = {{ buildout['develop-eggs-directory'] }}
offline = true
[instance-parameter]
recipe = slapos.cookbook:slapconfiguration
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[slap-configuration]
# frontend reads from from a part named [slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
partition = ${slap-connection:partition-id}
url = ${slap-connection:server-url}
key = ${slap-connection:key-file}
cert = ${slap-connection:cert-file}
[directory]
recipe = slapos.cookbook:mkdirectory
home = ${buildout:directory}
etc = ${:home}/etc
var = ${:home}/var
srv = ${:home}/srv
tmp = ${:home}/tmp
var-log = ${:var}/log
service = ${:etc}/service
promise = ${:etc}/promise
rest-server-data-dir = ${:srv}/restic
backup-caucased = ${:srv}/backup/caucased/
# Macros
[check-port-listening-promise]
recipe = slapos.cookbook:check_port_listening
path = ${directory:promise}/${:_buildout_section_name_}
[check-url-available-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promise}/${:_buildout_section_name_}
dash_path = {{ dash_bin }}
curl_path = {{ curl_bin }}
# Caucase
[rest-server-certificate]
key-file = ${directory:etc}/${:_buildout_section_name_}.key
cert-file = ${directory:etc}/${:_buildout_section_name_}.crt
common-name = ${:_buildout_section_name_}
ca-file = ${directory:etc}/${:_buildout_section_name_}.ca.crt
crl-file = ${directory:etc}/${:_buildout_section_name_}.crl
{{
caucase.updater(
prefix='rest-server-certificate',
buildout_bin_directory=buildout['bin-directory'],
updater_path='${directory:service}/rest-server-certificate-updater',
url='${caucased:url}',
data_dir='${directory:srv}/caucase-updater',
crt_path='${rest-server-certificate:cert-file}',
ca_path='${rest-server-certificate:ca-file}',
crl_path='${rest-server-certificate:crl-file}',
key_path='${rest-server-certificate:key-file}',
template_csr='${rest-server-certificate-prepare-csr:csr}',
openssl=openssl_bin,
)}}
[rest-server-certificate-csr-config]
recipe = slapos.recipe.template:jinja2
mode = 644
template = inline:
[req]
prompt = no
req_extensions = req_ext
distinguished_name = dn
[ dn ]
CN = restic-rest-server
[ req_ext ]
subjectAltName = @alt_names
[ alt_names ]
IP.1 = ${instance-parameter:ipv4-random}
IP.2 = ${instance-parameter:ipv6-random}
rendered = ${buildout:parts-directory}/${:_buildout_section_name_}/${:_buildout_section_name_}.txt
[rest-server-certificate-prepare-csr]
recipe = plone.recipe.command
command =
if [ ! -f '${:csr}' ] ; then
{{ openssl_bin }} req \
-newkey rsa:2048 \
-batch \
-new \
-nodes \
-keyout /dev/null \
-config '${rest-server-certificate-csr-config:rendered}' \
-out '${:csr}'
fi
stop-on-error = true
csr = ${directory:srv}/${:_buildout_section_name_}.csr.pem
[caucased]
port = 18080
ip = ${instance-parameter:ipv6-random}
netloc = [${:ip}]:${:port}
url = http://${:netloc}/
{{
caucase.caucased(
prefix='caucased',
buildout_bin_directory=buildout['bin-directory'],
caucased_path='${directory:service}/caucased',
backup_dir='${directory:backup-caucased}',
data_dir='${directory:srv}/caucased',
netloc='${caucased:netloc}',
tmp='${directory:tmp}',
service_auto_approve_count=1,
user_auto_approve_count=0,
key_len=2048,
)}}
[rest-server-password]
recipe = slapos.cookbook:generate.password
user = backup
[rest-server-htpassword]
recipe = plone.recipe.command
command =
if [ ! -f '${:htpassword}' ] ; then
{{ htpasswd_bin }} \
-b \
-B \
-c ${:htpassword} \
${rest-server-password:user} \
${rest-server-password:passwd}
fi
htpassword = ${directory:rest-server-data-dir}/.htpasswd
stop-on-error = true
[rest-server]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/rest-server \
--listen [${instance-parameter:ipv6-random}]:${:port}
--log ${directory:var-log}/${:_buildout_section_name_}-access.log
--path ${directory:rest-server-data-dir}
--tls
--tls-cert ${rest-server-certificate:cert-file}
--tls-key ${rest-server-certificate:key-file}
--prometheus
wrapper-path = ${directory:service}/rest-server
port = 19080
ip = ${instance-parameter:ipv6-random}
url = https://[${:ip}]:${:port}
depends =
${rest-server-htpassword:recipe}
[rest-server-listen-promise]
<= check-port-listening-promise
hostname= ${rest-server:ip}
port = ${rest-server:port}
[frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Rest Server Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${rest-server:url}
return = domain secure_access
[frontend-available-promise]
<= check-url-available-promise
url = ${frontend:connection-secure_access}
check-secure = 1
[promises]
recipe =
instance-promises =
${caucased-promise:recipe}
${rest-server-certificate-promise:recipe}
${rest-server-listen-promise:path}
${frontend-available-promise:path}
[publish-connection-parameter]
recipe = slapos.cookbook:publish
rest-server-user = ${rest-server-password:user}
rest-server-password = ${rest-server-password:passwd}
url = ${frontend:connection-secure_access}
caucase-url = ${caucased:url}
[buildout]
extends =
../../component/openssl/buildout.cfg
../../component/curl/buildout.cfg
../../component/dash/buildout.cfg
../../component/golang/buildout.cfg
../../component/restic/buildout.cfg
../../stack/caucase/buildout.cfg
../../stack/slapos.cfg
buildout.hash.cfg
parts =
slapos-cookbook
caucase-eggs
instance.cfg.in
restic
gowork
[python]
part = python3
[gowork]
install +=
${git.github.com_restic_rest-server:location}:./cmd/...
[git.github.com_restic_rest-server]
<= go-git-package
go.importpath = github.com/restic/rest-server
repository = https://github.com/restic/rest-server
revision = v0.10.0-0-g9313f19
[instance.cfg.in]
recipe = slapos.recipe.template:jinja2
rendered = ${buildout:directory}/instance.cfg
template = ${:_profile_base_location_}/${:filename}
mode = 0644
context =
section buildout buildout
key gowork_bin gowork:bin
raw openssl_bin ${openssl:location}/bin/openssl
raw htpasswd_bin ${apache:location}/bin/htpasswd
raw dash_bin ${dash:location}/bin/dash
raw curl_bin ${curl:location}/bin/curl
key template_monitor monitor2-template:rendered
import-list =
file caucase caucase-jinja2-library:target
Tests for restic ReST Server software release
##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from setuptools import setup, find_packages
version = '0.0.1.dev0'
name = 'slapos.test.restic_rest_server'
long_description = open("README.md").read()
setup(
name=name,
version=version,
description="Test for SlapOS' restic Rest Server",
long_description=long_description,
long_description_content_type='text/markdown',
maintainer="Nexedi",
maintainer_email="info@nexedi.com",
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.libnetworkcache',
'requests',
],
zip_safe=True,
test_suite='test',
)
##############################################################################
#
# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import glob
import os
import subprocess
import tempfile
import urllib.parse
import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestResticRestServer(SlapOSInstanceTestCase):
def setUp(self):
self.connection_parameters = \
self.computer_partition.getConnectionParameterDict()
parsed_url = urllib.parse.urlparse(self.connection_parameters['url'])
self.url_with_credentials = parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['rest-server-user'],
self.connection_parameters['rest-server-password'],
parsed_url.hostname,
parsed_url.port,
)).geturl()
self.ca_cert = self._getCaucaseServiceCACertificate()
def _getCaucaseServiceCACertificate(self):
ca_cert = tempfile.NamedTemporaryFile(
prefix="ca.crt.pem",
mode="w",
delete=False,
)
ca_cert.write(
requests.get(
urllib.parse.urljoin(
self.connection_parameters['caucase-url'],
'/cas/crt/ca.crt.pem',
)).text)
self.addCleanup(os.unlink, ca_cert.name)
return ca_cert.name
def test_http_get(self):
resp = requests.get(self.connection_parameters['url'], verify=self.ca_cert)
self.assertEqual(resp.status_code, requests.codes.unauthorized)
resp = requests.get(
urllib.parse.urljoin(
self.url_with_credentials,
'/metrics',
),
verify=self.ca_cert,
)
# a random metric
self.assertIn('process_cpu_seconds_total', resp.text)
resp.raise_for_status()
def test_backup_scenario(self):
restic_bin = os.path.join(
self.computer_partition_root_path,
'software_release',
'go.work',
'bin',
'restic',
)
def get_restic_output(*args, **kw):
return subprocess.check_output(
(
restic_bin,
'--cacert',
self.ca_cert,
'--password-file',
password_file.name,
'--repo',
'rest:' + self.url_with_credentials,
) + args,
universal_newlines=True,
**kw,
)
with tempfile.TemporaryDirectory() as work_directory,\
tempfile.NamedTemporaryFile(mode='w') as password_file:
password_file.write('secret')
password_file.flush()
with open(os.path.join(work_directory, 'data'), 'w') as f:
f.write('data to backup')
with self.assertRaises(subprocess.CalledProcessError) as exc_context:
get_restic_output('snapshots', stderr=subprocess.PIPE)
self.assertIn('Is there a repository at the following location?',
exc_context.exception.stderr)
out = get_restic_output('init')
self.assertIn('created restic repository', out)
out = get_restic_output('backup', work_directory)
self.assertIn('Added to the repo', out)
out = get_restic_output('snapshots')
self.assertEqual(out.splitlines()[-1], '1 snapshots')
snapshot_id = out.splitlines()[2].split()[0]
backup_path = out.splitlines()[2].split()[-1]
restore_directory = os.path.join(work_directory, 'restore')
out = get_restic_output(
'restore',
snapshot_id,
'--target',
restore_directory,
)
self.assertIn('restoring <Snapshot', out)
with open(os.path.join(restore_directory, backup_path, 'data')) as f:
self.assertEqual(f.read(), 'data to backup')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment