Commit 71793936 authored by Kirill Smelkov's avatar Kirill Smelkov

Merge branch 'x/lte-multiru' into xy/lte-multiru

- generic: publish primary information about RU and cells
- generic: fix instantiation for only NR cells
- ors: unconditionally test for tx_gain/rx_gain in enb.cfg
- misc...
parents d0a55b7b 582ea074
......@@ -44,9 +44,9 @@ environment =
[apache]
recipe = slapos.recipe.cmmi
shared = true
version = 2.4.57
version = 2.4.58
url = https://archive.apache.org/dist/httpd/httpd-${:version}.tar.bz2
md5sum = 34575e92f91fb625f3842d6ca382caa5
md5sum = 30377ec4d7fb8361e1d1f2ab3158b467
configure-options = --disable-static
--enable-authn-alias
--enable-bucketeer
......
[buildout]
extends =
../icu/buildout.cfg
../openssl/buildout.cfg
../pkgconfig/buildout.cfg
../readline/buildout.cfg
../zlib/buildout.cfg
../ncurses/buildout.cfg
......@@ -30,10 +32,13 @@ configure-options =
--without-libxslt
# build core PostgreSQL + pg_trgm contrib extension for GitLab
# unaccent contrib extension is for peertube
make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install
# citext contrib extension is for metabase
make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install && make -C contrib/citext/ install
environment =
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${perl:location}/libs-c -Wl,-rpath=${perl:location}/libs-c
PATH=${pkgconfig:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/include
LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${perl:location}/libs-c -Wl,-rpath=${perl:location}/libs-c -Wl,-rpath=${icu:location}/lib
PKG_CONFIG_PATH=${icu:location}/lib/pkgconfig/
[postgresql10]
<= postgresql-common
......
......@@ -9,8 +9,8 @@ parts = rustc
[rustc]
recipe = slapos.recipe.cmmi
shared = true
url = https://static.rust-lang.org/dist/rustc-1.64.0-src.tar.gz
md5sum = 948ecb62b82ed3543f03ebf598501796
url = https://static.rust-lang.org/dist/rustc-1.75.0-src.tar.gz
md5sum = 6420f7d4e0eb3f4f4963a2e2afdb301b
# --sysconfdir is a workaround for https://github.com/rust-lang/rust/issues/63915
configure-options =
--enable-extended
......
......@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '1.0.329'
version = '1.0.351'
name = 'slapos.cookbook'
long_description = open("README.rst").read()
......@@ -72,6 +72,8 @@ setup(name=name,
'zc.buildout', # plays with buildout
'zc.recipe.egg', # for scripts generation
'pytz', # for timezone database
'passlib',
'bcrypt',
],
zip_safe=True,
entry_points={
......
......@@ -24,9 +24,14 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import logging
import os
import time
from inotify_simple import INotify, flags
logger = logging.getLogger(__name__)
def subfiles(directory):
"""Return the list of subfiles of a directory, and wait for the newly created
ones.
......@@ -35,10 +40,19 @@ def subfiles(directory):
ALWAYS ITERATE OVER IT !!!*"""
with INotify() as inotify:
try:
inotify.add_watch(directory, flags.CLOSE_WRITE | flags.MOVED_TO)
inotify_available = True
except OSError:
logger.warning("Unable to add inotify watch, falling back to polling")
inotify_available = False
names = os.listdir(directory)
while True:
for name in names:
yield os.path.join(directory, name)
if inotify_available:
names = (event.name for event in inotify.read())
else:
time.sleep(5)
names = os.listdir(directory)
......@@ -131,7 +131,9 @@ class Recipe(GenericSlapRecipe):
new = {}
for k, v in six.iteritems(init):
try:
options[k] = publish_dict[k] = new[v] = init_section.pop(v)
init_section_value = init_section[v]
options[k] = publish_dict[k] = new[v] = init_section_value
del init_section[v]
except KeyError:
pass
if new != override:
......
......@@ -33,12 +33,16 @@ buildout Software Releases and Instances developments.
from __future__ import absolute_import
import errno
import json
import os
import random
import string
import sys
from .librecipe import GenericBaseRecipe
from .publish_early import volatileOptions
from slapos.util import str2bytes
import passlib.hash
class Integer(object):
"""
......@@ -113,7 +117,7 @@ def generatePassword(length):
class Password(object):
"""Generate a password that is only composed of lowercase letters
"""Generate a password.
This recipe only makes sure that ${:passwd} does not end up in `.installed`
file, which is world-readable by default. So be careful not to spread it
......@@ -128,6 +132,11 @@ class Password(object):
- create-once: boolean value which set if storage-path won't be modified
as soon the file is created with the password (not empty).
(default: True)
- passwd: the generated password. Can also be set, to reuse the password
hashing capabilities.
- passwd-*: the hashed password, using schemes supported by passlib.
for example, passwd-sha256-crypt will expose the password hashed
with sha256 crypt algorithm.
If storage-path is empty, the recipe does not save the password, which is
fine it is saved by other means, e.g. using the publish-early recipe.
......@@ -141,24 +150,53 @@ class Password(object):
except KeyError:
self.storage_path = options['storage-path'] = os.path.join(
buildout['buildout']['parts-directory'], name)
passwd = options.get('passwd')
if not passwd:
passwd_dict = {
'': options.get('passwd')
}
if not passwd_dict['']:
if self.storage_path:
self._needs_migration = False
try:
with open(self.storage_path) as f:
passwd = f.read().strip('\n')
content = f.read().strip('\n')
# new format: the file contains password and hashes in json format
try:
passwd_dict = json.loads(content)
if sys.version_info < (3, ):
passwd_dict = {k: v.encode() for k, v in passwd_dict.items()}
except ValueError:
# old format: the file only contains the password in plain text
passwd_dict[''] = content
self._needs_migration = True
except IOError as e:
if e.errno != errno.ENOENT:
raise
if not passwd:
passwd = self.generatePassword(int(options.get('bytes', '16')))
if not passwd_dict['']:
passwd_dict[''] = self.generatePassword(int(options.get('bytes', '16')))
self.update = self.install
options['passwd'] = passwd
options['passwd'] = passwd_dict['']
class HashedPasswordDict(dict):
def __missing__(self, key):
if not key.startswith('passwd-'):
raise KeyError(key)
if key in passwd_dict:
return passwd_dict[key]
handler = getattr(
passlib.hash, key[len('passwd-'):].replace('-', '_'), None)
if handler is None:
raise KeyError(key)
hashed = handler.hash(passwd_dict[''])
passwd_dict[key] = hashed
return hashed
options._data = HashedPasswordDict(options._data)
# Password must not go into .installed file, for 2 reasons:
# security of course but also to prevent buildout to always reinstall.
# publish_early already does it, but this recipe may also be used alone.
volatileOptions(options, ('passwd',))
self.passwd = passwd
self.passwd_dict = passwd_dict
generatePassword = staticmethod(generatePassword)
......@@ -167,19 +205,14 @@ class Password(object):
try:
# The following 2 lines are just an optimization to avoid recreating
# the file with the same content.
if self.create_once and os.stat(self.storage_path).st_size:
if self.create_once and os.stat(self.storage_path).st_size and not self._needs_migration:
return
os.unlink(self.storage_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
fd = os.open(self.storage_path,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0o600)
try:
os.write(fd, str2bytes(self.passwd))
finally:
os.close(fd)
with open(self.storage_path, 'w') as f:
json.dump(self.passwd_dict, f)
if not self.create_once:
return self.storage_path
......
import json
import os
import shutil
import tempfile
import unittest
import zc.buildout.testing
import zc.buildout.buildout
import passlib.hash
from slapos.recipe import random
class TestPassword(unittest.TestCase):
def setUp(self):
self.buildout = zc.buildout.testing.Buildout()
parts_directory = tempfile.mkdtemp()
self.buildout['buildout']['parts-directory'] = parts_directory
self.addCleanup(shutil.rmtree, parts_directory)
def _makeRecipe(self, options, section_name="random"):
self.buildout[section_name] = options
recipe = random.Password(
self.buildout, section_name, self.buildout[section_name]
)
return recipe
def test_empty_options(self):
recipe = self._makeRecipe({})
passwd = self.buildout["random"]["passwd"]
self.assertEqual(len(passwd), 16)
recipe.install()
with open(self.buildout["random"]["storage-path"]) as f:
self.assertEqual(json.load(f), {'': passwd})
def test_storage_path(self):
tf = tempfile.NamedTemporaryFile(delete=False)
self.addCleanup(os.unlink, tf.name)
self._makeRecipe({'storage-path': tf.name}).install()
passwd = self.buildout["random"]["passwd"]
self.assertEqual(len(passwd), 16)
with open(tf.name) as f:
self.assertEqual(json.load(f), {'': passwd})
self._makeRecipe({'storage-path': tf.name}, "another").install()
self.assertEqual(self.buildout["another"]["passwd"], passwd)
def test_storage_path_legacy_format(self):
with tempfile.NamedTemporaryFile(delete=False) as tf:
tf.write(b'secret\n')
tf.flush()
self._makeRecipe({'storage-path': tf.name}).install()
passwd = self.buildout["random"]["passwd"]
self.assertEqual(passwd, 'secret')
tf.flush()
with open(tf.name) as f:
self.assertEqual(json.load(f), {'': 'secret'})
self._makeRecipe({'storage-path': tf.name}, "another").install()
self.assertEqual(self.buildout["another"]["passwd"], passwd)
def test_bytes(self):
self._makeRecipe({'bytes': '32'}).install()
passwd = self.buildout["random"]["passwd"]
self.assertEqual(len(passwd), 32)
with open(self.buildout["random"]["storage-path"]) as f:
self.assertEqual(json.load(f), {'': passwd})
def test_volatile(self):
self._makeRecipe({})
options = self.buildout['random']
self.assertIn('passwd', options)
options_items = [(k, v) for k, v in options.items() if k != 'passwd']
copied_options = options.copy()
self.assertEqual(list(copied_options.items()), options_items)
def test_passlib(self):
recipe = self._makeRecipe({})
hashed = self.buildout['random']['passwd-sha256-crypt']
self.assertTrue(
passlib.hash.sha256_crypt.verify(
self.buildout['random']['passwd'], hashed))
hashed = self.buildout['random']['passwd-md5-crypt']
self.assertTrue(
passlib.hash.md5_crypt.verify(
self.buildout['random']['passwd'], hashed))
hashed = self.buildout['random']['passwd-bcrypt']
self.assertTrue(
passlib.hash.bcrypt.verify(
self.buildout['random']['passwd'], hashed))
hashed = self.buildout['random']['passwd-ldap-salted-sha1']
self.assertTrue(
passlib.hash.ldap_salted_sha1.verify(
self.buildout['random']['passwd'], hashed))
with self.assertRaises(zc.buildout.buildout.MissingOption):
self.buildout['random']['passwd-unknown']
with self.assertRaises(zc.buildout.buildout.MissingOption):
self.buildout['random']['unknown']
copied_options = self.buildout['random'].copy()
self.assertEqual(list(copied_options.keys()), ['storage-path'])
recipe.install()
# when buildout runs again, the values are read from the storage
# and even the hashed values are the same
self._makeRecipe({'storage-path': self.buildout['random']['storage-path']}, 'reread')
self.assertEqual(
self.buildout['reread']['passwd'],
self.buildout['random']['passwd'])
self.assertEqual(
self.buildout['reread']['passwd-sha256-crypt'],
self.buildout['random']['passwd-sha256-crypt'])
self.assertEqual(
self.buildout['reread']['passwd-bcrypt'],
self.buildout['random']['passwd-bcrypt'])
self.assertEqual(
self.buildout['reread']['passwd-ldap-salted-sha1'],
self.buildout['random']['passwd-ldap-salted-sha1'])
# values are strings which is important for python2
self.assertIsInstance(self.buildout['reread']['passwd'], str)
self.assertIsInstance(self.buildout['reread']['passwd-ldap-salted-sha1'], str)
def test_passlib_input_passwd(self):
self._makeRecipe({'passwd': 'insecure'})
self.assertEqual(self.buildout['random']['passwd'], 'insecure')
hashed = self.buildout['random']['passwd-sha256-crypt']
self.assertTrue(passlib.hash.sha256_crypt.verify('insecure', hashed))
......@@ -15,4 +15,4 @@
[instance.cfg.in]
filename = instance.cfg.in
md5sum = 6edf5c64bf25dfd2e6e8a4e74c9b9812
md5sum = f7a283e3288d3a9ddfd5de7e9b309273
......@@ -155,9 +155,10 @@ command-line =
--enable-cors
--bind ${:ip}
--port ${:port}
--render-try-index
--allow-all
--auth-method basic
--auth ${admin-password:user}:${admin-password:passwd}@/:rw
--auth ${admin-password:user}:${admin-password:passwd-sha512-crypt}@/:rw
--auth @/pub
--tls-cert ${dufs-certificate:cert-file}
--tls-key ${dufs-certificate:key-file}
......@@ -210,7 +211,7 @@ init =
frontend_url = urlparse(self.options['frontend-url'])
admin_user = self.options['admin-user']
admin_password = self.options['admin-password']
self.options['public-url'] = frontend_url._replace(path='/pub').geturl()
self.options['public-url'] = frontend_url._replace(path='/pub/').geturl()
assert not frontend_url.username
self.options['upload-url'] = frontend_url._replace(
netloc=f'{admin_user}:{admin_password}@{frontend_url.netloc}').geturl()
......
......@@ -13,8 +13,8 @@ parts =
[dufs]
recipe = slapos.recipe.cmmi
shared = true
url = https://github.com/sigoden/dufs/archive/refs/tags/v0.34.1.tar.gz
md5sum = 77cbb2523aca8dad90fd77ee0277704f
url = https://github.com/sigoden/dufs/archive/refs/tags/v0.39.0.tar.gz
md5sum = e0fca35530c043c5dff7fcbe415ed35d
configure-command = :
make-binary = cargo install --root=%(location)s --path . --locked
make-targets =
......
......@@ -91,6 +91,26 @@ class TestFileServer(SlapOSInstanceTestCase):
self.assertNotIn('secret', resp.text)
self.assertEqual(resp.status_code, requests.codes.ok)
def test_index(self):
pub = pathlib.Path(self.computer_partition_root_path) / 'srv' / 'www' / 'pub'
(pub / 'with-index').mkdir()
(pub / 'with-index' / 'index.html').write_text('<html>Hello !</html>')
self.assertEqual(
requests.get(
urllib.parse.urljoin(self.connection_parameters['public-url'], 'with-index/'),
verify=self.ca_cert,
).text,
'<html>Hello !</html>')
(pub / 'without-index').mkdir()
(pub / 'without-index' / 'file.txt').write_text('Hello !')
self.assertIn(
'file.txt',
requests.get(
urllib.parse.urljoin(self.connection_parameters['public-url'], 'without-index/'),
verify=self.ca_cert,
).text)
def test_upload_file_refused_without_auth(self):
parsed_upload_url = urllib.parse.urlparse(self.connection_parameters['upload-url'])
# upload-url has username:password, remove it
......
......@@ -32,6 +32,7 @@ import glob
import http.client
import json
import os
import pathlib
import resource
import shutil
import socket
......@@ -1100,6 +1101,22 @@ class TestNEO(ZopeSkinsMixin, CrontabMixin, ERP5InstanceTestCase):
'log',
f))
class TestPassword(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
__partition_reference__ = 'p'
def test_no_plain_text_password_in_files(self):
inituser_password = self.getRootPartitionConnectionParameterDict()[
'inituser-password'].encode()
self.assertFalse(
[f for f in pathlib.Path(self.slap._instance_root).glob('**/*')
if f.is_file() and inituser_password in f.read_bytes()])
# the hashed password is present in some files
inituser_password_hashed = self.getRootPartitionConnectionParameterDict()[
'inituser-password-hashed'].encode()
self.assertTrue(
[f for f in pathlib.Path(self.slap._instance_root).glob('**/*')
if f.is_file() and inituser_password_hashed in f.read_bytes()])
class TestWithMaxRlimitNofileParameter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
"""Test setting the with-max-rlimit-nofile parameter sets the open fd soft limit to the hard limit.
......
[instance-profile]
filename = instance.cfg.in
md5sum = 0d50ed911a41b76b952b63d37853c3a4
md5sum = f753802ad631a57c559d868e525cf81b
......@@ -3,7 +3,7 @@
"type": "object",
"additionalProperties": false,
"properties": {
"mb_password_complexity": {
"mb-password-complexity": {
"title": "Password complexity",
"description": "Check Metabase documentation for more details.",
"type": "string",
......@@ -14,11 +14,25 @@
"strong"
]
},
"mb_password_length": {
"mb-password-length": {
"title": "Password length",
"description": "Password length",
"type": "integer",
"default": 6
},
"mb-aggregated-query-row-limit": {
"title": "Aggregated row limit",
"description": "Maximum number of rows to return for aggregated queries via the API.",
"type": "integer",
"default": 10000,
"maximum": 1048575
},
"mb-unaggregated-query-row-limit": {
"title": "Unaggregated row limit",
"description": "Maximum number of rows to return specifically on `:rows`-type queries via the API. Must be less than the number configured in `mb-aggregated-query-row-limit`",
"type": "integer",
"default": 2000,
"maximum": 1048575
}
}
}
......@@ -31,8 +31,16 @@ slapparameter-dict = $${slap-configuration:configuration}
home = $${buildout:directory}
init =
default_parameters = options.get('slapparameter-dict')
options['mb_password_complexity'] = default_parameters.get('mb_password_complexity', 'normal')
options['mb_password_length'] = default_parameters.get('mb_password_length', '6')
options['mb-password-complexity'] = default_parameters.get(
'mb-password-complexity',
default_parameters.get('mb_password_complexity', 'normal'))
options['mb-password-length'] = default_parameters.get(
'mb-password-length',
default_parameters.get('mb_password_length', '6'))
options['mb-aggregated-query-row-limit'] = default_parameters.get(
'mb-aggregated-query-row-limit', '10000')
options['mb-unaggregated-query-row-limit'] = default_parameters.get(
'mb-unaggregated-query-row-limit', '2000')
[metabase-instance]
recipe = slapos.cookbook:wrapper
......@@ -51,8 +59,10 @@ environment =
MB_DB_USER=$${postgresql:superuser}
MB_DB_PASS=$${postgresql:password}
MB_DB_HOST=$${postgresql:ipv4}
MB_PASSWORD_COMPLEXITY=$${slap-parameter:mb_password_complexity}
MB_PASSWORD_LENGTH=$${slap-parameter:mb_password_length}
MB_PASSWORD_COMPLEXITY=$${slap-parameter:mb-password-complexity}
MB_PASSWORD_LENGTH=$${slap-parameter:mb-password-length}
MB_AGGREGATED_QUERY_ROW_LIMIT=$${slap-parameter:mb-aggregated-query-row-limit}
MB_UNAGGREGATED_QUERY_ROW_LIMIT=$${slap-parameter:mb-unaggregated-query-row-limit}
FONTCONFIG_FILE=$${fontconfig-conf:output}
JAVA_ARGS=-Dorg.quartz.scheduler.instanceId=$${slap-connection:computer-id}.$${slap-connection:partition-id} -Djava.io.tmpdir="$${directory:tmp}"
hash-existing-files =
......
......@@ -20,8 +20,8 @@ parts =
[metabase.jar]
recipe = slapos.recipe.build:download
url = https://downloads.metabase.com/v0.47.0/metabase.jar
md5sum = b81c71668a2177d89690730fabd85d9e
url = https://downloads.metabase.com/v0.48.2/metabase.jar
md5sum = d708a85436da3d5751f0e48ebd10c142
[instance-profile]
recipe = slapos.recipe.template
......
......@@ -36,7 +36,7 @@ md5sum = d2fe2fbe70c6824c514a7c4034a2ddb3
[ru_libinstance.jinja2.cfg]
_update_hash_filename_ = ru/libinstance.jinja2.cfg
md5sum = 39031a3b7f043c905b49986984d49a4b
md5sum = 803d273588bf930aa528f25a0b990804
[ru_sdr_libinstance.jinja2.cfg]
_update_hash_filename_ = ru/sdr/libinstance.jinja2.cfg
......@@ -88,7 +88,7 @@ md5sum = 52da9fe3a569199e35ad89ae1a44c30e
[template-enb]
_update_hash_filename_ = instance-enb.jinja2.cfg
md5sum = 51ce7e6fd96f3c8ad086723722b91c3f
md5sum = 98153725add6d377989c4d50b2054dd2
[template-ors-enb]
_update_hash_filename_ = instance-ors-enb.jinja2.cfg
......@@ -112,7 +112,7 @@ md5sum = dcaac06553a3222b14c0013a13f4a149
[enb.jinja2.cfg]
filename = config/enb.jinja2.cfg
md5sum = 5637fa338ffbc1015b5b6349ede91170
md5sum = b0d21e7b2c34f457e230d1cc950ce7fb
[drb_lte.jinja2.cfg]
filename = config/drb_lte.jinja2.cfg
......
......@@ -225,9 +225,9 @@
en_dc_support: true,
{%- endif %}
{% if do_lte %}
// LTE cells
cell_list: [
{%- if do_lte %}
{%- for i, (cell_ref, icell) in enumerate(icell_dict|dictsort) %}
{%- set cell = icell['_'] %}
{%- if cell.cell_type == 'lte' %}
......@@ -282,6 +282,9 @@
// tune LTE parameters for the cell
{%- if ors %}
manual_ref_signal_power: true,
{%- endif %}
{%- set tdd = (cell.rf_mode == 'tdd') %}
{%- if tdd %}
uldl_config: {{
......@@ -345,10 +348,6 @@
srs_hopping_bandwidth: 0,
},
{%- if ors %}
manual_ref_signal_power: true,
{%- endif %}
drb_config: "{{ B('%s-drb.cfg' % cell_ref) }}",
sib_sched_list: [
......@@ -360,8 +359,10 @@
},
{%- endif %}
{%- endfor %}
{%- endif %}
],
{%- if do_lte %}
cell_default: {
plmn_list: [
{%- for _, plmn in slapparameter_dict.plmn_list |dictsort %}
......@@ -399,7 +400,6 @@
pusch_msg3_mcs: 0,
dl_256qam: true,
ul_64qam: true,
......@@ -407,7 +407,6 @@
cqi_period: 40,
mac_config: {
ul_max_harq_tx: 5,
dl_max_harq_tx: 5,
......@@ -457,9 +456,9 @@
},
{%- endif %}
{% if do_nr %}
// NR cells
nr_cell_list: [
{%- if do_nr %}
{%- for i, (cell_ref, icell) in enumerate(icell_dict|dictsort) %}
{%- set cell = icell['_'] %}
{%- if cell.cell_type == 'nr' %}
......@@ -749,8 +748,10 @@
},
{%- endif %}
{%- endfor %}
{%- endif %}
],
{%- if do_nr %}
nr_cell_default: {
ssb_period: 20,
......
......@@ -40,7 +40,6 @@
gnb_id_bits: 28,
en_dc_support: true,
// LTE cells
cell_list: [
......@@ -148,7 +147,6 @@
],
},
],
cell_default: {
plmn_list: [
{
......@@ -184,7 +182,6 @@
pusch_msg3_mcs: 0,
dl_256qam: true,
ul_64qam: true,
......@@ -192,7 +189,6 @@
cqi_period: 40,
mac_config: {
ul_max_harq_tx: 5,
dl_max_harq_tx: 5,
......@@ -241,7 +237,6 @@
ho_from_meas: true,
},
// NR cells
nr_cell_list: [
......@@ -449,7 +444,6 @@
},
],
nr_cell_default: {
ssb_period: 20,
......
......@@ -7,5 +7,9 @@ git diff -C -C $@ -- \
':!test/test.sh' \
':!*.json' \
':!*.json.jinja2' \
':!test/*FDD*' \
':!test/*TDD*' \
':!*-fdd*' \
':!*-tdd*' \
':!diff.sh' \
......@@ -249,6 +249,9 @@ cell-list = {{ dumps(iru_icell_ref_list) }}
{%- if ru.ru_link_type == 'cpri' %}
ipv6 = ${vtap.{{ ru.cpri_link._tap }}:gateway}
{%- endif %}
tx_gain = {{ dumps(ru.tx_gain) }}
rx_gain = {{ dumps(ru.rx_gain) }}
txrx_active = {{ dumps(ru.txrx_active) }}
{%- endif %}
{%- endfor %}
......@@ -298,6 +301,19 @@ recipe = slapos.cookbook:publish.serialised
-slave-reference = {{ dumps(icell.slave_reference) }}
{{ slap_configuration['slap-software-type'] }} = {{ dumps(root) }}
ru = {{ dumps(ru_ref) }}
{%- if cell.cell_type == 'lte' %}
band = {{ dumps('b%d' % xearfcn_module.band(cell.dl_earfcn)[0].band) }}
dl_earfcn = {{ dumps(cell.dl_earfcn) }}
ul_earfcn = {{ dumps(cell.ul_earfcn) }}
{%- elif cell.cell_type == 'nr' %}
band = {{ dumps('n%d' % cell.nr_band) }}
dl_nr_arfcn = {{ dumps(cell.dl_nr_arfcn) }}
ul_nr_arfcn = {{ dumps(cell.ul_nr_arfcn) }}
ssb_nr_arfcn= {{ dumps(cell.ssb_nr_arfcn) }}
{%- else %}
{%- do bug('unreachable') %}
{%- endif %}
{%- endif %}
{%- endfor %}
......
......@@ -32,7 +32,7 @@ def j2render(src, out, jcfg):
textctx += 'import netaddr netaddr\n'
buildout = None # stub
r = jinja2_template.Recipe(buildout, "recipe", {
'extensions': 'jinja2.ext.do jinja2.ext.loopcontrols',
'extensions': 'jinja2.ext.do',
'url': 'config/{}'.format(src),
'output': 'config/out/{}'.format(out),
'context': textctx,
......@@ -555,6 +555,7 @@ def do_ue():
def main():
os.makedirs('config/out', exist_ok=True)
for f in glob.glob('config/out/*'):
os.remove(f)
do_enb()
......
......@@ -163,14 +163,35 @@ class AmariTestCase(_AmariTestCase):
# requestShared requests one shared instance over imain with specified subreference and parameters.
@classmethod
def requestShared(cls, imain, subref, ctx):
cls.slap.request(
ref = cls.ref(subref)
kw = dict(
software_release=cls.getSoftwareURL(),
software_type=cls.getInstanceSoftwareType(),
partition_reference=cls.ref(subref),
partition_reference=ref,
# XXX StandaloneSlapOS rejects filter_kw with "Can only request on embedded computer"
#filter_kw = {'instance_guid': imain.getInstanceGuid()},
partition_parameter_kw={'_': json.dumps(ctx)},
shared=True)
cls._requested[ref] = kw
return cls.slap.request(**kw)
# XXX StandaloneSlapOS lacks getInformation - we remember the way instances are requested ourselves.
_requested = {} # ref -> kw used for slap.request
# queryPiblished and querySharedPublished return information published by
# an instance / shared instance correspondingly.
@classmethod
def querySharedPublished(cls, subref):
return cls.queryPiblished(cls.ref(subref))
@classmethod
def queryPiblished(cls, ref):
# see ^^^ about lack of getInformation on StandaloneSlapOS
#inst = cls.slap.getInformation(computer_partition=ref)
inst = cls.slap.request(**cls._requested[ref])
iconn = inst.getConnectionParameterDict()
return json.loads(iconn['_'])
# ref returns full reference of shared instance with given subreference.
#
......@@ -225,7 +246,7 @@ class RFTestCase4(AmariTestCase):
ru = cls.RUcfg(i)
ru |= {'n_antenna_dl': 4, 'n_antenna_ul': 2}
ru |= {'tx_gain': 10+i, 'rx_gain': 20+i, 'txrx_active': 'ACTIVE'}
cls.requestShared(imain, 'RU%d' % i, ru)
return cls.requestShared(imain, 'RU%d' % i, ru)
def CELL(i, ctx):
cell = {
......@@ -236,7 +257,7 @@ class RFTestCase4(AmariTestCase):
}
cell |= cls.CELLcfg(i)
cell |= ctx
cls.requestShared(imain, 'RU%d.CELL' % i, cell)
return cls.requestShared(imain, 'RU%d.CELL' % i, cell)
RU(1); CELL(1, FDD | LTE( 100) | BW( 5))
RU(2); CELL(2, TDD | LTE( 40200) | BW(10))
......@@ -251,6 +272,26 @@ class RFTestCase4(AmariTestCase):
t.assertEqual(t.rf_cfg['tx_gain'], [11]*4 + [12]*4 + [13]*4 + [14]*4)
t.assertEqual(t.rf_cfg['rx_gain'], [21]*2 + [22]*2 + [23]*2 + [24]*2)
def test_published_ru_and_cell(t):
q = t.querySharedPublished
assertMatch(t, q('RU1'), {'tx_gain': 11, 'rx_gain': 21, 'txrx_active': 'ACTIVE'})
assertMatch(t, q('RU2'), {'tx_gain': 12, 'rx_gain': 22, 'txrx_active': 'ACTIVE'})
assertMatch(t, q('RU3'), {'tx_gain': 13, 'rx_gain': 23, 'txrx_active': 'ACTIVE'})
assertMatch(t, q('RU4'), {'tx_gain': 14, 'rx_gain': 24, 'txrx_active': 'ACTIVE'})
assertMatch(t, q('RU1.CELL'), dict(band='b1',
dl_earfcn= 100, ul_earfcn=18100,
dl_nr_arfcn=NO, ul_nr_arfcn=NO, ssb_nr_arfcn=NO))
assertMatch(t, q('RU2.CELL'), dict(band='b41',
dl_earfcn=40200, ul_earfcn=40200,
dl_nr_arfcn=NO, ul_nr_arfcn=NO, ssb_nr_arfcn=NO))
assertMatch(t, q('RU3.CELL'), dict(band='n74',
dl_earfcn=NO, ul_earfcn=NO,
dl_nr_arfcn=300300, ul_nr_arfcn=290700, ssb_nr_arfcn=300270))
assertMatch(t, q('RU4.CELL'), dict(band='n40',
dl_earfcn=NO, ul_earfcn=NO,
dl_nr_arfcn=470400, ul_nr_arfcn=470400, ssb_nr_arfcn=470430))
# ENBTestCase4 provides base class for unit-testing eNB service.
#
......@@ -664,7 +705,7 @@ class UEsimTestCase4(RFTestCase4):
'k': 'FFFF%028x' % i,
'impi': 'impi%d@rapid.space' % i,
}
cls.requestShared(imain, 'UE%d' % i, ue)
return cls.requestShared(imain, 'UE%d' % i, ue)
UE(1)
UE(2)
......
......@@ -142,11 +142,10 @@ def test_enb_conf(self):
self.slap.instance_directory, '*', 'etc', 'enb.cfg'))[0]
conf = yamlpp_load(conf_file)
if 'tx_gain' in conf and 'rx_gain' in conf:
self.assertEqual(conf['tx_gain'], [enb_param_dict['tx_gain']] * enb_param_dict['n_antenna_dl'])
self.assertEqual(conf['rx_gain'], [enb_param_dict['rx_gain']] * enb_param_dict['n_antenna_ul'])
self.assertEqual(len(conf['cell_list']), 1)
self.assertNotIn('nr_cell_list', conf)
self.assertEqual(conf['nr_cell_list'], [])
self.assertNotIn('nr_cell_default', conf)
cell = conf['cell_list'][0]
self.assertEqual(cell['inactivity_timer'], enb_param_dict['inactivity_timer'])
......@@ -183,7 +182,7 @@ def test_gnb_conf1(self):
conf = yamlpp_load(conf_file)
self.assertEqual(conf['tx_gain'], [gnb_param_dict1['tx_gain']] * gnb_param_dict1['n_antenna_dl'])
self.assertEqual(conf['rx_gain'], [gnb_param_dict1['rx_gain']] * gnb_param_dict1['n_antenna_ul'])
self.assertNotIn('cell_list', conf)
self.assertEqual(conf['cell_list'], [])
self.assertNotIn('cell_default', conf)
self.assertEqual(len(conf['nr_cell_list']), 1)
nr_cell = conf['nr_cell_list'][0]
......
......@@ -15,4 +15,4 @@
[instance.cfg.in]
filename = instance.cfg.in
md5sum = 361991f333119f22c8266dc8bde7bc57
md5sum = b5c479ebb4cf2fd2f63623af88b95078
......@@ -147,23 +147,15 @@ recipe = slapos.cookbook:generate.password
user = backup
[rest-server-htpassword]
recipe = plone.recipe.command
command =
if [ ! -f '${:htpassword}' ] ; then
{{ htpasswd_bin }} \
-b \
-B \
-c ${:htpassword} \
${rest-server-password:user} \
${rest-server-password:passwd}
fi
htpassword = ${directory:rest-server-data-dir}/.htpasswd
stop-on-error = true
recipe = slapos.recipe.template
inline =
${rest-server-password:user}:${rest-server-password:passwd-bcrypt}
output = ${directory:rest-server-data-dir}/.htpasswd
[rest-server]
recipe = slapos.cookbook:wrapper
command-line =
{{ gowork_bin }}/rest-server \
{{ gowork_bin }}/rest-server
--listen [${:ip}]:${:port}
--log ${directory:var-log}/${:_buildout_section_name_}-access.log
--path ${directory:rest-server-data-dir}
......
......@@ -25,7 +25,7 @@ install +=
<= go-git-package
go.importpath = github.com/restic/rest-server
repository = https://github.com/restic/rest-server
revision = v0.10.0-0-g9313f19
revision = v0.12.1-0-g4231ef6
[instance.cfg.in]
recipe = slapos.recipe.template:jinja2
......@@ -35,7 +35,6 @@ context =
section buildout buildout
key gowork_bin gowork:bin
raw openssl_bin ${openssl:location}/bin/openssl
raw htpasswd_bin ${apache:location}/bin/htpasswd
raw dash_bin ${dash:location}/bin/dash
raw curl_bin ${curl:location}/bin/curl
key template_monitor monitor2-template:output
......
......@@ -19,4 +19,4 @@ md5sum = 10e19df182c692b71ea552da183a0bcf
[template-selenium]
filename = instance-selenium.cfg.in
md5sum = 5a7abfff9f9d7898620f8c7fc1e6f488
\ No newline at end of file
md5sum = 7239845e758b2d10299699e061b0fc75
\ No newline at end of file
......@@ -170,14 +170,14 @@ content =
use_backend admin if { path_beg $${selenium-server-frontend-configuration:path-admin} }
userlist hub
user $${selenium-server-selenium-password:username} insecure-password $${selenium-server-selenium-password:passwd}
user $${selenium-server-selenium-password:username} password $${selenium-server-selenium-password:passwd-sha256-crypt}
backend hub
acl auth_ok http_auth(hub)
http-request auth realm "Selenium Server" unless auth_ok
server hub $${selenium-server-hub-instance:hostname}:$${selenium-server-hub-instance:port}
userlist admin
user $${selenium-server-admin-password:username} insecure-password $${selenium-server-admin-password:passwd}
user $${selenium-server-admin-password:username} password $${selenium-server-admin-password:passwd-sha256-crypt}
backend admin
acl auth_ok http_auth(admin)
http-request auth realm "Grid Admin" unless auth_ok
......
......@@ -14,7 +14,7 @@
# not need these here).
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 1fbfca2d64a9824054f7a3281e71efdc
md5sum = ba46a66da1c834df14a80a20b21e4a96
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -247,7 +247,7 @@ config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval'))
config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }}
config-zope-longrequest-logger-maximum-delay = {{ dumps(monitor_dict.get('zope-longrequest-logger-maximum-delay', 0)) }}
config-inituser-login = {{ dumps(inituser_login) }}
config-inituser-password = ${publish-early:inituser-password}
config-inituser-password-hashed = ${publish-early:inituser-password-hashed}
config-kumofs-url = ${request-memcached-persistent:connection-url}
config-memcached-url = ${request-memcached-volatile:connection-url}
config-monitor-passwd = ${monitor-htpasswd:passwd}
......@@ -515,6 +515,7 @@ hosts-dict = {{ '${' ~ zope_address_list_id_dict.keys()[0] ~ ':connection-hosts-
recipe = slapos.cookbook:publish-early
-init =
inituser-password gen-password:passwd
inituser-password-hashed gen-password:passwd-ldap-salted-sha1
deadlock-debugger-password gen-deadlock-debugger-password:passwd
{%- if has_posftix %}
smtpd-sasl-password gen-smtpd-sasl-password:passwd
......@@ -532,10 +533,6 @@ recipe = slapos.cookbook:publish-early
neo-cluster = {{ dumps(neo[0]) }}
{%- endif %}
{%- endif %}
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
inituser-password = {{ dumps(inituser_password) }}
{%- endif %}
{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
{%- if deadlock_debugger_password %}
deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
......@@ -552,6 +549,10 @@ recipe =
[gen-password]
recipe = slapos.cookbook:generate.password
storage-path =
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
passwd = {{ dumps(inituser_password) }}
{%- endif %}
[gen-deadlock-debugger-password]
<= gen-password
......
......@@ -15,7 +15,7 @@
[instance-theia]
_update_hash_filename_ = instance-theia.cfg.jinja.in
md5sum = 43b3435b3bc08db42335c03b5b8fe465
md5sum = 07b222d8c29d446fc0957e4e37706585
[instance]
_update_hash_filename_ = instance.cfg.in
......
......@@ -255,7 +255,7 @@ context =
key content :content
content =
userlist basic-auth-list
user $${frontend-instance-password:username} insecure-password $${frontend-instance-password:passwd}
user $${frontend-instance-password:username} password $${frontend-instance-password:passwd-sha256-crypt}
frontend app
log global
......
......@@ -40,9 +40,8 @@ import netaddr
import pexpect
import psutil
import requests
import six
from six.moves.urllib.parse import urlparse, urljoin
from urllib.parse import urlparse, urljoin, parse_qsl
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass, SlapOSNodeCommandError
from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
......@@ -621,7 +620,34 @@ class ResilientTheiaMixin(object):
class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
pass
def test_all_monitor_url_use_same_password(self):
monitor_setup_params = dict(
parse_qsl(
urlparse(
self.computer_partition.getConnectionParameterDict()
['monitor-setup-url']).fragment))
monitor_url_list = [
u for u in [
p.getConnectionParameterDict().get('monitor-base-url')
for p in self.slap.computer.getComputerPartitionList()
] if u is not None
]
self.assertEqual(len(monitor_url_list), 4)
for url in monitor_url_list:
self.assertEqual(
requests.get(url, verify=False).status_code,
requests.codes.unauthorized)
requests.get(
url,
verify=False,
auth=(
monitor_setup_params['username'],
monitor_setup_params['password'],
)).raise_for_status()
class TestTheiaResilientWithEmbeddedInstance(ResilientTheiaMixin, TestTheiaWithEmbeddedInstance):
......
......@@ -70,11 +70,11 @@ md5sum = b95084ae9eed95a68eada45e28ef0c04
[template]
filename = instance.cfg.in
md5sum = 55232eae0bcdb68a7cb2598d2ba9d60c
md5sum = 5e0e9565227fe190c420a7bbcd0f7b93
[template-erp5]
filename = instance-erp5.cfg.in
md5sum = 359bab24aec7772adb5d822c1389b1bd
md5sum = 2b91528d3a77a64714e4295a84c1d71b
[template-zeo]
filename = instance-zeo.cfg.in
......@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
[template-zope]
filename = instance-zope.cfg.in
md5sum = 2439b90d6f707f47050fc9074fa4d810
md5sum = 41709f47e5a9051ca4a9c943859f589b
[template-balancer]
filename = instance-balancer.cfg.in
......
......@@ -254,7 +254,7 @@ config-id-store-interval = {{ dumps(slapparameter_dict.get('id-store-interval'))
config-zope-longrequest-logger-error-threshold = {{ dumps(monitor_dict.get('zope-longrequest-logger-error-threshold', 20)) }}
config-zope-longrequest-logger-maximum-delay = {{ dumps(monitor_dict.get('zope-longrequest-logger-maximum-delay', 0)) }}
config-inituser-login = {{ dumps(inituser_login) }}
config-inituser-password = ${publish-early:inituser-password}
config-inituser-password-hashed = ${publish-early:inituser-password-hashed}
config-kumofs-url = ${request-memcached-persistent:connection-url}
config-memcached-url = ${request-memcached-volatile:connection-url}
config-monitor-passwd = ${monitor-htpasswd:passwd}
......@@ -515,6 +515,7 @@ hosts-dict = {{ '${' ~ next(iter(zope_address_list_id_dict)) ~ ':connection-host
recipe = slapos.cookbook:publish-early
-init =
inituser-password gen-password:passwd
inituser-password-hashed gen-password:passwd-ldap-salted-sha1
deadlock-debugger-password gen-deadlock-debugger-password:passwd
{%- if has_posftix %}
smtpd-sasl-password gen-smtpd-sasl-password:passwd
......@@ -532,10 +533,6 @@ recipe = slapos.cookbook:publish-early
neo-cluster = {{ dumps(neo[0]) }}
{%- endif %}
{%- endif %}
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
inituser-password = {{ dumps(inituser_password) }}
{%- endif %}
{%- set deadlock_debugger_password = slapparameter_dict.get('deadlock-debugger-password') -%}
{%- if deadlock_debugger_password %}
deadlock-debugger-password = {{ dumps(deadlock_debugger_password) }}
......@@ -552,6 +549,10 @@ recipe =
[gen-password]
recipe = slapos.cookbook:generate.password
storage-path =
{%- set inituser_password = slapparameter_dict.get('inituser-password') %}
{%- if inituser_password %}
passwd = {{ dumps(inituser_password) }}
{%- endif %}
[gen-deadlock-debugger-password]
<= gen-password
......
......@@ -252,7 +252,7 @@ file-list = {{ parameter_dict['site-zcml'] }}
[{{ section('zope-inituser') }}]
< = jinja2-template-base
output = ${directory:instance}/inituser
inline = {{ slapparameter_dict['inituser-login'] }}:{SHA}{{ base64.b64encode(hashlib.sha1(slapparameter_dict['inituser-password'].encode('utf-8')).digest()) }}
inline = {{ slapparameter_dict['inituser-login'] }}:{{ slapparameter_dict['inituser-password-hashed'] }}
once = ${:output}_done
[zope-conf-parameter-base]
......
......@@ -143,9 +143,7 @@ extra-context =
key buildout_directory buildout:directory
key root_common context:root-common
section parameter_dict dynamic-template-zope-parameters
import base64 base64
import urllib_parse six.moves.urllib.parse
import hashlib hashlib
import itertools itertools
import json json
import-list =
......
......@@ -14,7 +14,7 @@
# not need these here).
[monitor2-template]
filename = instance-monitor.cfg.jinja2.in
md5sum = 3850140a4e61349cc64fa924ce410803
md5sum = 24c7f5527d994e231b4c2bf9fecb68a6
[monitor-httpd-conf]
_update_hash_filename_ = templates/monitor-httpd.conf.in
......
......@@ -112,7 +112,6 @@ parameter-list =
htpasswd monitor-password ${httpd-monitor-htpasswd:password-file} ${monitor-instance-parameter:username} ${httpd-monitor-htpasswd:htpasswd-path}
file min-free-disk-MB ${promise-check-free-disk-space:config-threshold-file}
${monitor-instance-parameter:instance-configuration}
# htpasswd entry: htpasswd key password-file username htpasswd-file
promise-output-file = ${directory:monitor}/monitor-bootstrap-status
......@@ -157,14 +156,11 @@ storage-path = ${directory:etc}/.monitor_pwd
[httpd-monitor-htpasswd]
recipe = plone.recipe.command
stop-on-error = true
password-file = ${directory:etc}/.monitor_pwd
password-file = ${monitor-directory:etc}/.monitor-password
htpasswd-path = ${monitor-directory:etc}/monitor-htpasswd
command =
echo "${monitor-instance-parameter:password}" >${:password-file}
[ -s "${:htpasswd-path}" ] ||
{{ apache_location }}/bin/htpasswd -ci ${:htpasswd-path} "${monitor-instance-parameter:username}" <${:password-file}
update-command =
[ -s "${:password-file}" ] || ${:command}
{{ apache_location }}/bin/htpasswd -cib ${:htpasswd-path} "${monitor-instance-parameter:username}" "${monitor-instance-parameter:password}"
[monitor-symlink]
recipe = cns.recipe.symlink
......@@ -343,8 +339,6 @@ collector-db = /srv/slapgrid/var/data-log/collector.db
# Credentials
password = ${monitor-htpasswd:passwd}
username = admin
# XXX: type key value
# ex raw monitor-password resqdsdsd34
instance-configuration =
configuration-file-path = ${monitor-directory:etc}/monitor_knowledge0.cfg
......
......@@ -20,6 +20,7 @@ allow-picked-versions = false
# Use shacache and lxml
extends =
../component/defaults.cfg
../component/bcrypt/buildout.cfg
../component/git/buildout.cfg
../component/lxml-python/buildout.cfg
../component/python-backports-lzma/buildout.cfg
......@@ -102,6 +103,7 @@ eggs =
${python-PyYAML:egg}
${python-cryptography:egg}
${pyrsistent:egg}
${bcrypt:egg}
# slapos.toolbox containing utilities
[slapos-toolbox]
......@@ -302,8 +304,8 @@ sgmllib3k = 1.0.0
simplegeneric = 0.8.1
singledispatch = 3.4.0.3
six = 1.16.0
slapos.cookbook = 1.0.329
slapos.core = 1.10.8
slapos.cookbook = 1.0.351
slapos.core = 1.11.0
slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.25
slapos.rebootstrap = 4.5
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment