Merge branch 'master' into x/lte-multiru

* master:
  fixup! software/ors-amarisoft: Merge gnb.jinja2.cfg into enb.jinja2.cfg
  Release slapos.cookbook (1.0.351)
  random: expose hashed passwords in recipe options
  librecipe/inotify: fallback to polling when adding inotify watch fail
  software/theia/test: test that monitoring URL all have same password
  software/metabase: version up 0.48.2
  software/metabase: rename configuration parameters to kebab case
  component/postgresql: include citext extension

component/postgresql/buildout.cfg

 [buildout]
-extends = 
+extends =
+  ../icu/buildout.cfg
   ../openssl/buildout.cfg
+  ../pkgconfig/buildout.cfg
   ../readline/buildout.cfg
   ../zlib/buildout.cfg
   ../ncurses/buildout.cfg
@@ -30,10 +32,13 @@ configure-options =
   --without-libxslt
 # build core PostgreSQL + pg_trgm contrib extension for GitLab
 # unaccent contrib extension is for peertube
-make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install
+# citext contrib extension is for metabase
+make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install && make -C contrib/citext/ install
 environment =
-  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib
-  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${perl:location}/libs-c -Wl,-rpath=${perl:location}/libs-c
+  PATH=${pkgconfig:location}/bin:%(PATH)s
+  CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/include
+  LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${perl:location}/libs-c -Wl,-rpath=${perl:location}/libs-c -Wl,-rpath=${icu:location}/lib
+  PKG_CONFIG_PATH=${icu:location}/lib/pkgconfig/
 
 [postgresql10]
 <= postgresql-common

setup.py

@@ -28,7 +28,7 @@ from setuptools import setup, find_packages
 import glob
 import os
 
-version = '1.0.329'
+version = '1.0.351'
 name = 'slapos.cookbook'
 long_description = open("README.rst").read()
 
@@ -72,6 +72,8 @@ setup(name=name,
         'zc.buildout', # plays with buildout
         'zc.recipe.egg', # for scripts generation
         'pytz', # for timezone database
+        'passlib',
+        'bcrypt',
         ],
       zip_safe=True,
       entry_points={

slapos/recipe/librecipe/inotify.py

@@ -24,9 +24,14 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
+import logging
 import os
+import time
+
 from inotify_simple import INotify, flags
 
+logger = logging.getLogger(__name__)
+
 def subfiles(directory):
   """Return the list of subfiles of a directory, and wait for the newly created
   ones.
@@ -35,10 +40,19 @@ def subfiles(directory):
   ALWAYS ITERATE OVER IT !!!*"""
 
   with INotify() as inotify:
-    inotify.add_watch(directory, flags.CLOSE_WRITE | flags.MOVED_TO)
+    try:
+      inotify.add_watch(directory, flags.CLOSE_WRITE | flags.MOVED_TO)
+      inotify_available = True
+    except OSError:
+      logger.warning("Unable to add inotify watch, falling back to polling")
+      inotify_available = False
 
     names = os.listdir(directory)
     while True:
       for name in names:
         yield os.path.join(directory, name)
-      names = (event.name for event in inotify.read())
+      if inotify_available:
+        names = (event.name for event in inotify.read())
+      else:
+        time.sleep(5)
+        names = os.listdir(directory)

slapos/recipe/publish_early.py

@@ -131,7 +131,9 @@ class Recipe(GenericSlapRecipe):
           new = {}
           for k, v in six.iteritems(init):
             try:
-              options[k] = publish_dict[k] = new[v] = init_section.pop(v)
+              init_section_value = init_section[v]
+              options[k] = publish_dict[k] = new[v] = init_section_value
+              del init_section[v]
             except KeyError:
               pass
           if new != override:

slapos/recipe/random.py

@@ -33,12 +33,16 @@ buildout Software Releases and Instances developments.
 from __future__ import absolute_import
 
 import errno
+import json
 import os
 import random
 import string
+import sys
 from .librecipe import GenericBaseRecipe
 from .publish_early import volatileOptions
-from slapos.util import str2bytes
+
+import passlib.hash
+
 
 class Integer(object):
   """
@@ -113,7 +117,7 @@ def generatePassword(length):
 
 
 class Password(object):
-  """Generate a password that is only composed of lowercase letters
+  """Generate a password.
 
     This recipe only makes sure that ${:passwd} does not end up in `.installed`
     file, which is world-readable by default. So be careful not to spread it
@@ -128,6 +132,11 @@ class Password(object):
     - create-once: boolean value which set if storage-path won't be modified
                    as soon the file is created with the password (not empty).
       (default: True)
+    - passwd: the generated password. Can also be set, to reuse the password
+              hashing capabilities.
+    - passwd-*: the hashed password, using schemes supported by passlib.
+                for example, passwd-sha256-crypt will expose the password hashed
+                with sha256 crypt algorithm.
 
     If storage-path is empty, the recipe does not save the password, which is
     fine it is saved by other means, e.g. using the publish-early recipe.
@@ -141,24 +150,53 @@ class Password(object):
     except KeyError:
       self.storage_path = options['storage-path'] = os.path.join(
         buildout['buildout']['parts-directory'], name)
-    passwd = options.get('passwd')
-    if not passwd:
+    passwd_dict = {
+      '': options.get('passwd')
+    }
+    if not passwd_dict['']:
       if self.storage_path:
+        self._needs_migration = False
         try:
           with open(self.storage_path) as f:
-            passwd = f.read().strip('\n')
+            content = f.read().strip('\n')
+            # new format: the file contains password and hashes in json format
+            try:
+              passwd_dict = json.loads(content)
+              if sys.version_info < (3, ):
+                passwd_dict = {k: v.encode() for k, v in passwd_dict.items()}
+            except ValueError:
+              # old format: the file only contains the password in plain text
+              passwd_dict[''] = content
+              self._needs_migration = True
         except IOError as e:
           if e.errno != errno.ENOENT:
             raise
-      if not passwd:
-        passwd = self.generatePassword(int(options.get('bytes', '16')))
+
+      if not passwd_dict['']:
+        passwd_dict[''] = self.generatePassword(int(options.get('bytes', '16')))
         self.update = self.install
-      options['passwd'] = passwd
+      options['passwd'] = passwd_dict['']
+
+    class HashedPasswordDict(dict):
+      def __missing__(self, key):
+        if not key.startswith('passwd-'):
+          raise KeyError(key)
+        if key in passwd_dict:
+          return passwd_dict[key]
+        handler = getattr(
+          passlib.hash, key[len('passwd-'):].replace('-', '_'), None)
+        if handler is None:
+          raise KeyError(key)
+        hashed = handler.hash(passwd_dict[''])
+        passwd_dict[key] = hashed
+        return hashed
+    options._data = HashedPasswordDict(options._data)
+
     # Password must not go into .installed file, for 2 reasons:
     # security of course but also to prevent buildout to always reinstall.
     # publish_early already does it, but this recipe may also be used alone.
     volatileOptions(options, ('passwd',))
-    self.passwd = passwd
+    self.passwd_dict = passwd_dict
 
   generatePassword = staticmethod(generatePassword)
 
@@ -167,19 +205,14 @@ class Password(object):
       try:
         # The following 2 lines are just an optimization to avoid recreating
         # the file with the same content.
-        if self.create_once and os.stat(self.storage_path).st_size:
+        if self.create_once and os.stat(self.storage_path).st_size and not self._needs_migration:
           return
         os.unlink(self.storage_path)
       except OSError as e:
         if e.errno != errno.ENOENT:
           raise
-
-      fd = os.open(self.storage_path,
-        os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0o600)
-      try:
-        os.write(fd, str2bytes(self.passwd))
-      finally:
-        os.close(fd)
+      with open(self.storage_path, 'w') as f:
+        json.dump(self.passwd_dict, f)
       if not self.create_once:
         return self.storage_path
 

slapos/test/recipe/test_random.py

+import json
+import os
+import shutil
+import tempfile
+import unittest
+
+import zc.buildout.testing
+import zc.buildout.buildout
+import passlib.hash
+from slapos.recipe import random
+
+
+class TestPassword(unittest.TestCase):
+  def setUp(self):
+    self.buildout = zc.buildout.testing.Buildout()
+    parts_directory = tempfile.mkdtemp()
+    self.buildout['buildout']['parts-directory'] = parts_directory
+    self.addCleanup(shutil.rmtree, parts_directory)
+
+  def _makeRecipe(self, options, section_name="random"):
+    self.buildout[section_name] = options
+    recipe = random.Password(
+      self.buildout, section_name, self.buildout[section_name]
+    )
+    return recipe
+
+  def test_empty_options(self):
+    recipe = self._makeRecipe({})
+    passwd = self.buildout["random"]["passwd"]
+    self.assertEqual(len(passwd), 16)
+    recipe.install()
+    with open(self.buildout["random"]["storage-path"]) as f:
+      self.assertEqual(json.load(f), {'': passwd})
+
+  def test_storage_path(self):
+    tf = tempfile.NamedTemporaryFile(delete=False)
+    self.addCleanup(os.unlink, tf.name)
+    self._makeRecipe({'storage-path': tf.name}).install()
+    passwd = self.buildout["random"]["passwd"]
+    self.assertEqual(len(passwd), 16)
+    with open(tf.name) as f:
+      self.assertEqual(json.load(f), {'': passwd})
+
+    self._makeRecipe({'storage-path': tf.name}, "another").install()
+    self.assertEqual(self.buildout["another"]["passwd"], passwd)
+
+  def test_storage_path_legacy_format(self):
+    with tempfile.NamedTemporaryFile(delete=False) as tf:
+      tf.write(b'secret\n')
+      tf.flush()
+
+      self._makeRecipe({'storage-path': tf.name}).install()
+      passwd = self.buildout["random"]["passwd"]
+      self.assertEqual(passwd, 'secret')
+      tf.flush()
+      with open(tf.name) as f:
+        self.assertEqual(json.load(f), {'': 'secret'})
+
+    self._makeRecipe({'storage-path': tf.name}, "another").install()
+    self.assertEqual(self.buildout["another"]["passwd"], passwd)
+
+  def test_bytes(self):
+    self._makeRecipe({'bytes': '32'}).install()
+    passwd = self.buildout["random"]["passwd"]
+    self.assertEqual(len(passwd), 32)
+    with open(self.buildout["random"]["storage-path"]) as f:
+      self.assertEqual(json.load(f), {'': passwd})
+
+  def test_volatile(self):
+    self._makeRecipe({})
+    options = self.buildout['random']
+    self.assertIn('passwd', options)
+    options_items = [(k, v) for k, v in options.items() if k != 'passwd']
+    copied_options = options.copy()
+    self.assertEqual(list(copied_options.items()), options_items)
+
+  def test_passlib(self):
+    recipe = self._makeRecipe({})
+
+    hashed = self.buildout['random']['passwd-sha256-crypt']
+    self.assertTrue(
+      passlib.hash.sha256_crypt.verify(
+        self.buildout['random']['passwd'], hashed))
+
+    hashed = self.buildout['random']['passwd-md5-crypt']
+    self.assertTrue(
+      passlib.hash.md5_crypt.verify(
+        self.buildout['random']['passwd'], hashed))
+
+    hashed = self.buildout['random']['passwd-bcrypt']
+    self.assertTrue(
+      passlib.hash.bcrypt.verify(
+        self.buildout['random']['passwd'], hashed))
+
+    hashed = self.buildout['random']['passwd-ldap-salted-sha1']
+    self.assertTrue(
+      passlib.hash.ldap_salted_sha1.verify(
+        self.buildout['random']['passwd'], hashed))
+
+    with self.assertRaises(zc.buildout.buildout.MissingOption):
+      self.buildout['random']['passwd-unknown']
+    with self.assertRaises(zc.buildout.buildout.MissingOption):
+      self.buildout['random']['unknown']
+
+    copied_options = self.buildout['random'].copy()
+    self.assertEqual(list(copied_options.keys()), ['storage-path'])
+
+    recipe.install()
+    # when buildout runs again, the values are read from the storage
+    # and even the hashed values are the same
+    self._makeRecipe({'storage-path': self.buildout['random']['storage-path']}, 'reread')
+    self.assertEqual(
+      self.buildout['reread']['passwd'],
+      self.buildout['random']['passwd'])
+    self.assertEqual(
+      self.buildout['reread']['passwd-sha256-crypt'],
+      self.buildout['random']['passwd-sha256-crypt'])
+    self.assertEqual(
+      self.buildout['reread']['passwd-bcrypt'],
+      self.buildout['random']['passwd-bcrypt'])
+    self.assertEqual(
+      self.buildout['reread']['passwd-ldap-salted-sha1'],
+      self.buildout['random']['passwd-ldap-salted-sha1'])
+    # values are strings which is important for python2
+    self.assertIsInstance(self.buildout['reread']['passwd'], str)
+    self.assertIsInstance(self.buildout['reread']['passwd-ldap-salted-sha1'], str)
+
+  def test_passlib_input_passwd(self):
+    self._makeRecipe({'passwd': 'insecure'})
+    self.assertEqual(self.buildout['random']['passwd'], 'insecure')
+
+    hashed = self.buildout['random']['passwd-sha256-crypt']
+    self.assertTrue(passlib.hash.sha256_crypt.verify('insecure', hashed))

software/metabase/buildout.hash.cfg

 [instance-profile]
 filename = instance.cfg.in
-md5sum = 0d50ed911a41b76b952b63d37853c3a4
+md5sum = f753802ad631a57c559d868e525cf81b

software/metabase/instance-metabase-input-schema.json

@@ -3,7 +3,7 @@
   "type": "object",
   "additionalProperties": false,
   "properties": {
-    "mb_password_complexity": {
+    "mb-password-complexity": {
       "title": "Password complexity",
       "description": "Check Metabase documentation for more details.",
       "type": "string",
@@ -14,11 +14,25 @@
         "strong"
       ]
     },
-    "mb_password_length": {
+    "mb-password-length": {
       "title": "Password length",
       "description": "Password length",
       "type": "integer",
       "default": 6
+    },
+    "mb-aggregated-query-row-limit": {
+      "title": "Aggregated row limit",
+      "description": "Maximum number of rows to return for aggregated queries via the API.",
+      "type": "integer",
+      "default": 10000,
+      "maximum": 1048575
+    },
+    "mb-unaggregated-query-row-limit": {
+      "title": "Unaggregated row limit",
+      "description": "Maximum number of rows to return specifically on `:rows`-type queries via the API. Must be less than the number configured in `mb-aggregated-query-row-limit`",
+      "type": "integer",
+      "default": 2000,
+      "maximum": 1048575
     }
   }
 }

software/metabase/instance.cfg.in

@@ -31,8 +31,16 @@ slapparameter-dict = $${slap-configuration:configuration}
 home = $${buildout:directory}
 init =
   default_parameters = options.get('slapparameter-dict')
-  options['mb_password_complexity'] = default_parameters.get('mb_password_complexity', 'normal')
-  options['mb_password_length'] = default_parameters.get('mb_password_length', '6')
+  options['mb-password-complexity'] = default_parameters.get(
+    'mb-password-complexity',
+      default_parameters.get('mb_password_complexity', 'normal'))
+  options['mb-password-length'] = default_parameters.get(
+    'mb-password-length',
+      default_parameters.get('mb_password_length', '6'))
+  options['mb-aggregated-query-row-limit'] = default_parameters.get(
+    'mb-aggregated-query-row-limit', '10000')
+  options['mb-unaggregated-query-row-limit'] = default_parameters.get(
+    'mb-unaggregated-query-row-limit', '2000')
 
 [metabase-instance]
 recipe = slapos.cookbook:wrapper
@@ -51,8 +59,10 @@ environment =
   MB_DB_USER=$${postgresql:superuser}
   MB_DB_PASS=$${postgresql:password}
   MB_DB_HOST=$${postgresql:ipv4}
-  MB_PASSWORD_COMPLEXITY=$${slap-parameter:mb_password_complexity}
-  MB_PASSWORD_LENGTH=$${slap-parameter:mb_password_length}
+  MB_PASSWORD_COMPLEXITY=$${slap-parameter:mb-password-complexity}
+  MB_PASSWORD_LENGTH=$${slap-parameter:mb-password-length}
+  MB_AGGREGATED_QUERY_ROW_LIMIT=$${slap-parameter:mb-aggregated-query-row-limit}
+  MB_UNAGGREGATED_QUERY_ROW_LIMIT=$${slap-parameter:mb-unaggregated-query-row-limit}
   FONTCONFIG_FILE=$${fontconfig-conf:output}
   JAVA_ARGS=-Dorg.quartz.scheduler.instanceId=$${slap-connection:computer-id}.$${slap-connection:partition-id} -Djava.io.tmpdir="$${directory:tmp}"
 hash-existing-files =

software/metabase/software.cfg

@@ -20,8 +20,8 @@ parts =
 
 [metabase.jar]
 recipe = slapos.recipe.build:download
-url = https://downloads.metabase.com/v0.47.0/metabase.jar
-md5sum = b81c71668a2177d89690730fabd85d9e
+url = https://downloads.metabase.com/v0.48.2/metabase.jar
+md5sum = d708a85436da3d5751f0e48ebd10c142
 
 [instance-profile]
 recipe = slapos.recipe.template

software/theia/test/test.py

@@ -40,9 +40,8 @@ import netaddr
 import pexpect
 import psutil
 import requests
-import six
 
-from six.moves.urllib.parse import urlparse, urljoin
+from urllib.parse import urlparse, urljoin, parse_qsl
 
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass, SlapOSNodeCommandError
 from slapos.grid.svcbackend import getSupervisorRPC, _getSupervisordSocketPath
@@ -621,7 +620,34 @@ class ResilientTheiaMixin(object):
 
 
 class TestTheiaResilientInterface(ResilientTheiaMixin, TestTheia):
-  pass
+
+  def test_all_monitor_url_use_same_password(self):
+    monitor_setup_params = dict(
+      parse_qsl(
+        urlparse(
+          self.computer_partition.getConnectionParameterDict()
+          ['monitor-setup-url']).fragment))
+
+    monitor_url_list = [
+      u for u in [
+        p.getConnectionParameterDict().get('monitor-base-url')
+        for p in self.slap.computer.getComputerPartitionList()
+      ] if u is not None
+    ]
+    self.assertEqual(len(monitor_url_list), 4)
+
+    for url in monitor_url_list:
+      self.assertEqual(
+        requests.get(url, verify=False).status_code,
+        requests.codes.unauthorized)
+
+      requests.get(
+        url,
+        verify=False,
+        auth=(
+          monitor_setup_params['username'],
+          monitor_setup_params['password'],
+        )).raise_for_status()
 
 
 class TestTheiaResilientWithEmbeddedInstance(ResilientTheiaMixin, TestTheiaWithEmbeddedInstance):