Commit 7467cb0c authored by Joanne Hugé's avatar Joanne Hugé

Update Release Candidate

parents 19ebaf23 51ca4092
......@@ -48,8 +48,10 @@ environment =
[gdal-python]
recipe = zc.recipe.egg:custom
egg = GDAL ==${gdal:version}
egg = GDAL ==${gdal:version}+SlapOSPatched001
setup-eggs = ${numpy:egg}
patches = https://github.com/OSGeo/gdal/commit/166ee6fee9c5e2356605e89abf72c23f3bd0cb74.patch?full_index=1#878ccd1b33e03a5d7445e8d4ff9b6746
patch-options = -p3
rpath = ${:library-dirs}
include-dirs =
${gdal:location}/include
......
......@@ -2,6 +2,7 @@
extends =
../flex/buildout.cfg
../libnsl/buildout.cfg
../lz4/buildout.cfg
../lzo/buildout.cfg
../openssl/buildout.cfg
../xz-utils/buildout.cfg
......@@ -20,9 +21,7 @@ configure-options =
--enable-iproute2
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
LZO_LIBS=-L${lzo:location}/lib -llzo2
LZO_CFLAGS=-I${lzo:location}/include
OPENSSL_LIBS=-L${openssl-1.1:location}/lib -lcrypto -lssl
OPENSSL_CFLAGS=-I${openssl-1.1:location}/include
LDFLAGS =-Wl,-rpath=${lzo:location}/lib -Wl,-rpath=${flex:location}/lib -Wl,-rpath=${openssl-1.1:location}/lib -L${libnsl:location}/lib -Wl,-rpath=${libnsl:location}/lib
CPPFLAGS=-I${lzo:location}/include -I${flex:location}/include -I${libnsl:location}/include
LDFLAGS=-L${lz4:location}/lib -Wl,-rpath=${lz4:location}/lib -L${lzo:location}/lib -Wl,-rpath=${lzo:location}/lib -Wl,-rpath=${flex:location}/lib -Wl,-rpath=${openssl-1.1:location}/lib -L${libnsl:location}/lib -Wl,-rpath=${libnsl:location}/lib
CPPFLAGS=-I${lz4:location}/include -I${lzo:location}/include -I${flex:location}/include -I${libnsl:location}/include
......@@ -14,6 +14,11 @@ eggs = ${pytest:eggs}
[versions]
pytest = 8.3.3:whl
pytest-timeout = 2.3.1
pytest-mock = 3.14.0:whl
[versions:python2]
pytest = 4.6.11:whl
pytest-timeout = 1.4.2
pytest-mock = 2.0.0:whl
......@@ -18,8 +18,8 @@ parts =
[dependency-track-bundled.jar]
recipe = slapos.recipe.build:download
url = https://github.com/DependencyTrack/dependency-track/releases/download/4.11.4/dependency-track-bundled.jar
md5sum = a1c5e8f216a999b6d497b133c93588a6
url = https://github.com/DependencyTrack/dependency-track/releases/download/4.12.1/dependency-track-bundled.jar
md5sum = 3621372d92eab7d7e7e0302cbd697de6
[instance-profile]
recipe = slapos.recipe.template
......
......@@ -41,7 +41,7 @@ setup(name=name,
url="https://lab.nexedi.com/nexedi/slapos",
packages=find_packages(),
install_requires=[
'slapos.core',
'slapos.core[testing]',
'supervisor',
'slapos.libnetworkcache',
'erp5.util',
......
......@@ -25,26 +25,13 @@
#
##############################################################################
import hashlib
import itertools
import json
import os
import shutil
import subprocess
import sys
import tempfile
import time
import urllib
import requests
from cryptography import x509
from cryptography.hazmat.backends import default_backend
from cryptography.hazmat.primitives import hashes, serialization
from cryptography.hazmat.primitives.asymmetric import rsa
from cryptography.x509.oid import NameOID
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.testcase import ManagedResource, makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort
ERP5PY3 = os.environ['SLAPOS_SR_TEST_NAME'] == 'erp5-py3'
......@@ -221,174 +208,3 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase, metaclass=ERP5InstanceTestMet
def getComputerPartitionPath(cls, partition_reference):
partition_id = cls.getComputerPartition(partition_reference).getId()
return os.path.join(cls.slap._instance_root, partition_id)
class CaucaseService(ManagedResource):
"""A caucase service.
"""
url: str = None
directory: str = None
_caucased_process: subprocess.Popen = None
def open(self) -> None:
# start a caucased and server certificate.
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
caucased_path = os.path.join(software_release_root_path, 'bin', 'caucased')
self.directory = tempfile.mkdtemp()
caucased_dir = os.path.join(self.directory, 'caucased')
os.mkdir(caucased_dir)
os.mkdir(os.path.join(caucased_dir, 'user'))
os.mkdir(os.path.join(caucased_dir, 'service'))
backend_caucased_netloc = f'{self._cls._ipv4_address}:{findFreeTCPPort(self._cls._ipv4_address)}'
self.url = 'http://' + backend_caucased_netloc
self._caucased_process = subprocess.Popen(
[
caucased_path,
'--db', os.path.join(caucased_dir, 'caucase.sqlite'),
'--server-key', os.path.join(caucased_dir, 'server.key.pem'),
'--netloc', backend_caucased_netloc,
'--service-auto-approve-count', '1',
],
# capture subprocess output not to pollute test's own stdout
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
)
for _ in range(30):
try:
if requests.get(self.url).status_code == 200:
break
except Exception:
pass
time.sleep(1)
else:
raise RuntimeError('caucased failed to start.')
def close(self) -> None:
self._caucased_process.terminate()
self._caucased_process.wait()
self._caucased_process.stdout.close()
shutil.rmtree(self.directory)
@property
def ca_crt_path(self) -> str:
"""Path of the CA certificate from this caucase.
"""
ca_crt_path = os.path.join(self.directory, 'ca.crt.pem')
if not os.path.exists(ca_crt_path):
with open(ca_crt_path, 'w') as f:
f.write(
requests.get(urllib.parse.urljoin(
self.url,
'/cas/crt/ca.crt.pem',
)).text)
return ca_crt_path
class CaucaseCertificate(ManagedResource):
"""A certificate signed by a caucase service.
"""
ca_crt_file: str = None
crl_file: str = None
csr_file: str = None
cert_file: str = None
key_file: str = None
def open(self) -> None:
self.tmpdir = tempfile.mkdtemp()
self.ca_crt_file = os.path.join(self.tmpdir, 'ca-crt.pem')
self.crl_file = os.path.join(self.tmpdir, 'ca-crl.pem')
self.csr_file = os.path.join(self.tmpdir, 'csr.pem')
self.cert_file = os.path.join(self.tmpdir, 'crt.pem')
self.key_file = os.path.join(self.tmpdir, 'key.pem')
def close(self) -> None:
shutil.rmtree(self.tmpdir)
@property
def _caucase_path(self) -> str:
"""path of caucase executable.
"""
software_release_root_path = os.path.join(
self._cls.slap._software_root,
hashlib.md5(self._cls.getSoftwareURL().encode()).hexdigest(),
)
return os.path.join(software_release_root_path, 'bin', 'caucase')
def request(self, common_name: str, caucase: CaucaseService, san: x509.SubjectAlternativeName=None) -> None:
"""Generate certificate and request signature to the caucase service.
This overwrite any previously requested certificate for this instance.
"""
cas_args = [
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
]
key = rsa.generate_private_key(
public_exponent=65537,
key_size=2048,
backend=default_backend()
)
with open(self.key_file, 'wb') as f:
f.write(
key.private_bytes(
encoding=serialization.Encoding.PEM,
format=serialization.PrivateFormat.TraditionalOpenSSL,
encryption_algorithm=serialization.NoEncryption(),
))
csr = x509.CertificateSigningRequestBuilder().subject_name(
x509.Name([
x509.NameAttribute(
NameOID.COMMON_NAME,
common_name,
),
]))
if san:
csr = csr.add_extension(san, critical=True)
csr = csr.sign(key, hashes.SHA256(), default_backend())
with open(self.csr_file, 'wb') as f:
f.write(csr.public_bytes(serialization.Encoding.PEM))
csr_id = subprocess.check_output(
cas_args + [
'--send-csr', self.csr_file,
],
).split()[0].decode()
assert csr_id
for _ in range(30):
if not subprocess.call(
cas_args + [
'--get-crt', csr_id, self.cert_file,
],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
) == 0:
break
else:
time.sleep(1)
else:
raise RuntimeError('getting service certificate failed.')
with open(self.cert_file) as cert_file:
assert 'BEGIN CERTIFICATE' in cert_file.read()
def revoke(self, caucase: CaucaseService) -> None:
"""Revoke the client certificate on this caucase instance.
"""
subprocess.check_call([
self._caucase_path,
'--ca-url', caucase.url,
'--ca-crt', self.ca_crt_file,
'--crl', self.crl_file,
'--revoke-crt', self.cert_file, self.key_file,
])
......@@ -44,6 +44,9 @@ from . import ERP5InstanceTestCase, default, matrix, setUpModule
from .test_erp5 import ZopeSkinsMixin
_ = setUpModule
class TestOrderBuildPackingListSimulation(
ZopeSkinsMixin,
ERP5InstanceTestCase,
......
......@@ -23,11 +23,12 @@ import psutil
import requests
from slapos.proxy.db_version import DB_VERSION
from slapos.testing.caucase import CaucaseCertificate, CaucaseService
from slapos.testing.utils import CrontabMixin, ManagedHTTPServer
from . import CaucaseCertificate, CaucaseService, ERP5InstanceTestCase, default, matrix, setUpModule
from . import ERP5InstanceTestCase, default, matrix, setUpModule
setUpModule # pyflakes
_ = setUpModule
class EchoHTTPServer(ManagedHTTPServer):
......
......@@ -48,12 +48,14 @@ import xmlrpc.client
import psutil
import requests
import urllib3
from slapos.testing.caucase import CaucaseService
from slapos.testing.utils import CrontabMixin
import zc.buildout.configparser
from . import CaucaseService, ERP5InstanceTestCase, default, matrix, neo, setUpModule, ERP5PY3
setUpModule # pyflakes
from . import ERP5InstanceTestCase, default, matrix, neo, setUpModule, ERP5PY3
_ = setUpModule
class TestPublishedURLIsReachableMixin:
......
......@@ -42,7 +42,7 @@ from slapos.testing.utils import CrontabMixin, getPromisePluginParameterDict
from . import ERP5InstanceTestCase, default, matrix, setUpModule
setUpModule # pyflakes
_ = setUpModule
class MariaDBTestCase(ERP5InstanceTestCase):
......
......@@ -16,7 +16,7 @@
[template]
filename = instance.cfg
md5sum = f1b9ae02222c020c89d1a3fa75475826
md5sum = 770fccd0e4c5ad30e9637fd643a126c5
[template-ors]
filename = instance-ors.cfg
......@@ -60,7 +60,7 @@ md5sum = 52da9fe3a569199e35ad89ae1a44c30e
[template-enb]
_update_hash_filename_ = instance-enb.jinja2.cfg
md5sum = 9cd5250fc523863d206b8dc038beaf82
md5sum = a05c6ed1830d77a9faea67ac54cf9c2f
[template-ors-enb]
_update_hash_filename_ = instance-ors-enb.jinja2.cfg
......
......@@ -212,6 +212,12 @@
"type": "number",
"default": 0
},
"xlog_forwarding_enabled": {
"title": "Enable eNB xlog forwarding",
"description": "Enable eNB xlog forwarding through fluentbit",
"type": "boolean",
"default": "true"
},
"wendelin_telecom_software_release_url": {
"title": "Wendelin Telecom Software Release URL",
"description": "URL of the Wendelin Telecom Software Release to use to request a shared instance",
......@@ -221,8 +227,7 @@
"xlog_fluentbit_forward_host": {
"title": "Fluentbit Xlog forwarding address",
"description": "Address of remote Fluentd or Fluentbit server to which Fluentbit should forward Xlog data",
"type": "string",
"default": "fluentd.rapid.space"
"type": "string"
},
"xlog_fluentbit_forward_port": {
"title": "Fluentbit Xlog forwarding port",
......
......@@ -8,8 +8,8 @@
'use_ipv4': False,
'gnb_id_bits': 28,
'nssai': {'1': {'sst': 1}},
'xlog_forwarding_enabled': True,
'wendelin_telecom_software_release_url': 'wendelin-telecom-enb-shared-instance',
'xlog_fluentbit_forward_host': 'fluentd.rapid.space',
'xlog_fluentbit_forward_port': 24224,
} %}
{%- set gtp_addr_lo = '127.0.1.1' %}
......@@ -26,8 +26,12 @@ parts =
enb-config
enb-service
xamari-xlog-service
xlog-fluentbit-service
request-wendelin-telecom-shared
{%- if slapparameter_dict.xlog_forwarding_enabled %}
xlog-fluentbit-service
check-xlog-fluentbit-forward-host.py
check-xlog-fluentbit-health.py
{%- endif %}
check-baseband-latency.py
monitor-base
publish-connection-information
......@@ -165,11 +169,25 @@ wrapper-path = ${directory:service}/${:_buildout_section_name_}
command-line = ${xamari-xlog-script:output}
hash-files = ${:command-line}
[request-wendelin-telecom-shared]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Wendelin Telecom Registration
software-url = {{ slapparameter_dict.wendelin_telecom_software_release_url }}
shared = true
{%- if not slapparameter_dict.xlog_forwarding_enabled or slapparameter_dict.get("xlog_fluentbit_forward_host") %}
state = destroyed
{%- else %}
config-fluentbit-tag = ${xlog-fluentbit-tag:xlog-fluentbit-tag}
return = gateway-host
{%- endif %}
{% if slapparameter_dict.xlog_forwarding_enabled %}
[xlog-fluentbit-tag]
recipe = slapos.recipe.build
computer = ${slap-connection:computer-id}
enb-id = {{ slapparameter_dict.get("enb_id") }}
gnb-id = {{ slapparameter_dict.get("gnb_id") }}
enb-id = {{ slapparameter_dict.get("enb_id", "") }}
gnb-id = {{ slapparameter_dict.get("gnb_id", "") }}
init =
import socket
......@@ -189,13 +207,22 @@ init =
recipe = slapos.recipe.template
output = ${directory:etc}/${:_buildout_section_name_}.cfg
logfile = ${xamari-xlog-script:logfile}
forward-host = {{ slapparameter_dict.xlog_fluentbit_forward_host }}
forward-host = {{ slapparameter_dict.get('xlog_fluentbit_forward_host', '') or '${request-wendelin-telecom-shared:connection-gateway-host}' }}
forward-port = {{ slapparameter_dict.xlog_fluentbit_forward_port }}
forward-shared-key = {{ slapparameter_dict.get('xlog_fluentbit_forward_shared_key', '') }}
forward-self-hostname = {{ B(comp_id['comp-id']) }}
monitoring-host = {{ my_ipv4 }}
monitoring-port = {{ slapparameter_dict.xlog_fluentbit_forward_port + 1 }}
inline =
[SERVICE]
flush 5
HTTP_Server On
HTTP_Listen ${:monitoring-host}
HTTP_PORT ${:monitoring-port}
Health_Check On
HC_Errors_Count 0
HC_Retry_Failure_Count 0
HC_Period 60
[INPUT]
name tail
path ${:logfile}
......@@ -212,6 +239,7 @@ inline =
Shared_Key ${:forward-shared-key}
{%- endif %}
Self_Hostname ${:forward-self-hostname}
Retry_Limit 50
tls on
tls.verify off
......@@ -223,13 +251,17 @@ command-line = ${:fluentbit} -c ${:fluentbit-config}
wrapper-path = ${directory:service}/${:_buildout_section_name_}
hash-files = ${:fluentbit-config}
[request-wendelin-telecom-shared]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Wendelin Telecom Registration
software-url = {{ slapparameter_dict.wendelin_telecom_software_release_url }}
shared = true
config-fluentbit-tag = ${xlog-fluentbit-tag:xlog-fluentbit-tag}
[check-xlog-fluentbit-forward-host.py]
<= macro.promise
promise = check_socket_listening
config-host = ${xlog-fluentbit-config:forward-host}
config-port = ${xlog-fluentbit-config:forward-port}
[check-xlog-fluentbit-health.py]
<= macro.promise
promise = check_url_available
config-url = http://${xlog-fluentbit-config:monitoring-host}:${xlog-fluentbit-config:monitoring-port}/api/v1/health
{%- endif %}
[config-base]
recipe = slapos.recipe.template:jinja2
......@@ -280,7 +312,9 @@ ru-list = {{ dumps(rulib.iru_dict.keys() | sort) }}
cell-list = {{ dumps(rulib.icell_dict.keys() | sort) }}
peer-list = {{ dumps(ipeer_dict.keys() | sort) }}
peer-cell-list = {{ dumps(ipeercell_dict.keys() | sort) }}
{%- if slapparameter_dict.xlog_forwarding_enabled %}
fluentbit-tag = ${xlog-fluentbit-tag:xlog-fluentbit-tag}
{%- endif %}
[monitor-instance-parameter]
......
......@@ -199,6 +199,9 @@
"min_rxtx_delay": {
"$ref": "instance-enb-input-schema.json#/properties/min_rxtx_delay"
},
"xlog_forwarding_enabled": {
"$ref": "instance-enb-input-schema.json#/properties/xlog_forwarding_enabled"
},
"wendelin_telecom_software_release_url": {
"$ref": "instance-enb-input-schema.json#/properties/wendelin_telecom_software_release_url"
},
......
......@@ -189,6 +189,12 @@
"min_rxtx_delay": {
"$ref": "instance-ors-enb-input-schema.json#/properties/min_rxtx_delay"
},
"xlog_forwarding_enabled": {
"$ref": "instance-ors-enb-input-schema.json#/properties/xlog_forwarding_enabled"
},
"wendelin_telecom_software_release_url": {
"$ref": "instance-ors-enb-input-schema.json#/properties/wendelin_telecom_software_release_url"
},
"xlog_fluentbit_forward_host": {
"$ref": "instance-ors-enb-input-schema.json#/properties/xlog_fluentbit_forward_host"
},
......
......@@ -40,6 +40,7 @@ context =
section slap_connection slap-connection
key slapparameter_dict slap-configuration:configuration
key lan_ipv4 lan-ip:ipv4
key my_ipv4 slap-configuration:ipv4-random
key my_ipv6 slap-configuration:ipv6-random
$${:extra-context}
import-list =
......
......@@ -332,6 +332,7 @@ class ENBTestCase4(RFTestCase4):
'1': {'plmn': '51413', 'tac': 0x124},
'2': {'plmn': '55555', 'tac': 0x125, 'ranac': 210, 'reserved': True},
},
'xlog_forwarding_enabled': False,
})}
@classmethod
......
......@@ -104,6 +104,7 @@ enb_param_dict = {
'tac': 2
},
},
'xlog_forwarding_enabled': False,
}
gnb_param_dict = {
# ors_version for tests is B39, so dl_nr_arfcn needs to be within N39
......@@ -148,6 +149,7 @@ gnb_param_dict = {
'tac': 2
},
},
'xlog_forwarding_enabled': False,
}
gnb_param_dict1 = {
'plmn_list': {
......
......@@ -78,7 +78,7 @@ md5sum = 1333d2fc21f64da4010a4eafea59d141
[template-zeo]
filename = instance-zeo.cfg.in
md5sum = 6513f1ecd9a2daaf36ca720f15932ae3
md5sum = 025f11aef75bf00d61e90c6734ce13d8
[template-zeo-conf]
filename = zeo.conf.in
......
......@@ -108,9 +108,9 @@ inline =
--file="$zodb_directory/$zodb_path"
CURRENT_EXIT_CODE=$?
if [ ! "$CURRENT_EXIT_CODE"="0" ]; then
if [ $CURRENT_EXIT_CODE != 0 ]; then
EXIT_CODE="$CURRENT_EXIT_CODE"
echo "$storage_name Backup restoration failed."
echo "$storage_name Backup failed."
fi
{% endfor -%}
{% endfor -%}
......
......@@ -151,6 +151,7 @@ annotated-types = 0.6.0:whl
anyio = 4.3.0:whl
apache-libcloud = 2.4.0
argon2-cffi = 20.1.0
arrow = 1.2.3
asn1crypto = 1.3.0
astor = 0.8.1
asttokens = 2.4.1:whl
......@@ -209,6 +210,7 @@ executing = 2.0.1:whl
fastjsonschema = 2.18.1
feedparser = 6.0.10
Flask = 3.0.0:whl
fqdn = 1.5.1
frozenlist = 1.4.0:whl
funcsigs = 1.0.2
functools32 = 3.2.3.post2
......@@ -231,12 +233,14 @@ Importing = 1.10
importlib-metadata = 6.8.0:whl
importlib-resources = 5.10.2:whl
incremental = 22.10.0
iniconfig = 2.0.0:whl
inotify-simple = 1.1.1
ipaddress = 1.0.23
ipykernel = 6.29.3:whl
ipython = 8.18.1:whl
ipython-genutils = 0.2.0
ipywidgets = 8.1.2:whl
isoduration = 20.11.0
itsdangerous = 2.1.2
jdcal = 1.4
jedi = 0.17.2
......@@ -250,7 +254,6 @@ jupyter-client = 8.6.1:whl
jupyter-console = 6.6.3:whl
jupyter-core = 5.7.1:whl
jupyter-events = 0.6.3:whl
isoduration = 20.11.0
jupyter-lsp = 2.2.3:whl
jupyter-server = 2.10.0:whl
jupyter-server-terminals = 0.5.2:whl
......@@ -259,8 +262,6 @@ jupyterlab-launcher = 0.13.1
jupyterlab-pygments = 0.3.0:whl
jupyterlab-server = 2.24.0:whl
jupyterlab-widgets = 3.0.10:whl
arrow = 1.2.3
fqdn = 1.5.1
lock-file = 2.0
lockfile = 0.12.2:whl
lsprotocol = 2023.0.0b1:whl
......@@ -304,7 +305,7 @@ pkgconfig = 1.5.1:whl
pkgutil-resolve-name = 1.3.10:whl
platformdirs = 4.2.0:whl
plone.recipe.command = 1.1
pluggy = 0.13.1:whl
pluggy = 1.5.0:whl
ply = 3.11
prettytable = 0.7.2
prometheus-client = 0.9.0
......@@ -469,6 +470,7 @@ nbclient = 0.5.1
netaddr = 0.7.19
notebook = 6.1.5
packaging = 16.8
pluggy = 0.13.1:whl
psycopg2 = 2.8.6
pycurl = 7.43.0
pyparsing = 2.4.7
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment