Metabase

See merge request !700

software/metabase/README.md

+# Metabae
+
+https://www.metabase.com/
+
+## TODO:
+ * export backups for resilience
+ * security (proper passwords, verifiable certificate, study metabase encryption option)

software/metabase/buildout.hash.cfg

+[instance-profile]
+filename = instance.cfg.in
+md5sum = 8e48fa7c66a59b3d5faf0216922a574f

software/metabase/instance.cfg.in

+[buildout]
+parts =
+  publish-connection-parameter
+
+extends = ${monitor2-template:rendered}
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[metabase-instance]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:service}/$${:_buildout_section_name_}
+command-line = sh -c "cd $${directory:srv-metabase}; ${java-re-8:location}/bin/java $JAVA_ARGS -jar ${metabase.jar:location}/metabase.jar"
+# https://www.metabase.com/docs/latest/operations-guide/customizing-jetty-webserver.html
+# note that we set org.quartz.scheduler.instanceId through $JAVA_ARGS as a workaround for machines
+# which cannot resolve their hostnames. See also https://github.com/metabase/metabase/issues/8373
+environment =
+  MB_EMOJI_IN_LOGS=false
+  MB_JETTY_HOST=$${:ip}
+  MB_JETTY_PORT=$${:-http-port}
+  MB_JETTY_SSL_PORT=$${:port}
+  MB_JETTY_SSL=true
+  MB_JETTY_SSL_KEYSTORE=$${metabase-keystore:file}
+  MB_JETTY_SSL_KEYSTORE_PASSWORD=$${metabase-keystore:password}
+  MB_DB_TYPE=postgres
+  MB_DB_DBNAME=$${postgresql:dbname}
+  MB_DB_PORT=$${postgresql:port}
+  MB_DB_USER=$${postgresql:superuser}
+  MB_DB_PASS=$${postgresql:password}
+  MB_DB_HOST=$${postgresql:ipv4}
+  JAVA_ARGS=-Dorg.quartz.scheduler.instanceId=$${slap-connection:computer-id}.$${slap-connection:partition-id}
+hash-existing-files =
+  $${buildout:directory}/software_release/buildout.cfg
+
+ip = $${instance-parameter:ipv6-random}
+port = 8443
+# XXX It does not seem we can prevent metabase to also listen on http, so we
+# give it an http port, but don't use it.
+-http-port = 18080
+hostname = [$${:ip}]
+scheme = https
+url = $${:scheme}://$${:hostname}:$${:port}
+promises =
+  $${metabase-promise:name}
+
+[metabase-promise]
+<= monitor-promise-base
+module = check_url_available
+name = $${:_buildout_section_name_}.py
+config-url= $${metabase-instance:url}/api/session/properties
+
+[metabase-keystore]
+recipe = plone.recipe.command
+command =
+  ${java-re-8-output:keytool} \
+    -genkeypair \
+    -alias "metabase" \
+    -keyalg RSA \
+    -keypass "$${:password}" \
+    -dname "CN=$${metabase-instance:ip},OU=Unit,O=Organization,L=City,S=State,C=Country" \
+    -keystore "$${:file}" \
+    -storepass "$${:password}"
+file = $${directory:etc}/.metabase_keystore
+password = insecure
+
+[postgresql]
+recipe = slapos.cookbook:postgres
+bin = ${postgresql10:location}/bin/
+services = $${directory:service}
+dbname  = metabase_db
+superuser = metabase-psql
+password = insecure
+pgdata-directory = $${directory:srv}/postgresql
+
+ipv4 = $${instance-parameter:ipv4-random}
+# disable listening on ipv6
+ipv6 =
+port = 5432
+
+promises = $${postgresql-promise:name}
+
+
+[postgresql-psql]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
+command-line =
+    $${postgresql:bin}/psql
+        -h $${postgresql:pgdata-directory}
+        -U $${postgresql:superuser}
+        -d $${postgresql:dbname}
+
+[postgresql-promise]
+<= monitor-promise-base
+module = check_command_execute
+name = promise-postgresql.py
+config-command = $${postgresql-psql:wrapper-path} -c '\q'
+
+[postgresql-backup-crontab-entry]
+recipe = slapos.cookbook:cron.d
+name = $${:_buildout_section_name_}
+cron-entries = $${cron:cron-entries}
+time = daily
+command = $${postgresql-backup:wrapper-path}
+
+[postgresql-backup]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
+# XXX there's a recipe for backup in slapos cookbook, but it does not create
+# the backup file in an atomic way, which is not acceptable here, because we
+# don't want to risk pulling a partial file. To prevent this, we create a
+# temp file and move it when finished.
+command-line =
+    sh -c "$${postgresql:bin}/pg_dump \
+      -h $${postgresql:pgdata-directory} \
+      -U $${postgresql:superuser} \
+      --format=custom \
+      -f $${:backup-file}.tmp \
+      $${postgresql:dbname} \
+    && mv $${:backup-file}.tmp $${:backup-file}"
+backup-file = $${directory:srv-backup}/backup.pg_dump
+
+[postgresql-restore-backup]
+recipe = slapos.cookbook:wrapper
+wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
+command-line =
+  sh -e -c "\
+    echo 'This will replace current database with latest backup. Hit Ctrl+C to cancel';
+    sleep 5;
+    $${postgresql:bin}/pg_restore \
+        --exit-on-error \
+        -h $${postgresql:pgdata-directory} \
+        -U $${postgresql:superuser} \
+        -d $${postgresql:dbname} \
+        $${postgresql-backup:backup-file}"
+
+[cron]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron-output:crond}
+cron-entries = $${directory:etc-cron.d}
+crontabs = $${directory:var-crontabs}
+cronstamps = $${directory:var-cronstamps}
+catcher = $${cron-simplelogger:wrapper}
+binary = $${directory:bin}/crond
+
+[cron-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${cron:binary}
+wrapper-path = $${directory:services}/crond
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
+
+[cron-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${directory:bin}/cron_simplelogger
+log = $${directory:log}/cron.log
+
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = $${slap-connection:computer-id}
+partition = $${slap-connection:partition-id}
+url = $${slap-connection:server-url}
+key = $${slap-connection:key-file}
+cert = $${slap-connection:cert-file}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+etc-cron.d = $${:etc}/cron.d
+var = $${buildout:directory}/var
+var-crontabs = $${:var}/crontabs
+var-cronstamps = $${:var}/cronstamps
+var-cron-entries = $${:var}/cron-entries
+srv = $${buildout:directory}/srv
+bin = $${buildout:directory}/bin
+tmp = $${buildout:directory}/tmp
+service = $${:etc}/service
+srv-metabase = $${:srv}/metabase
+srv-backup = $${:srv}/backup
+
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+url = $${metabase-instance:url}
+backup-crontab = $${postgresql-backup-crontab-entry:name}
+restore-backup-script = $${postgresql-restore-backup:wrapper-path}

software/metabase/software.cfg

+[buildout]
+extends =
+    ../../component/defaults.cfg
+    ../../component/java/buildout.cfg
+    ../../component/postgresql/buildout.cfg
+    ../../component/dcron/buildout.cfg
+    ../../stack/slapos.cfg
+    buildout.hash.cfg
+    ../../stack/monitor/buildout.cfg
+
+parts =
+  slapos-cookbook
+  instance-profile
+
+[python]
+part = python3
+
+[metabase.jar]
+recipe = slapos.recipe.build:download
+url = https://downloads.metabase.com/v0.35.3/metabase.jar
+md5sum = 73c98cdf5cecde80463ef868e77d3b0e
+
+[instance-profile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/instance.cfg
+
+[versions]
+slapos.recipe.template = 4.4

software/metabase/test/README.md

+Tests for Metabase software release

software/metabase/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.metabase'
+with open("README.md") as f:
+  long_description = f.read()
+
+setup(name=name,
+      version=version,
+      description="Test for SlapOS' metabase",
+      long_description=long_description,
+      long_description_content_type='text/markdown',
+      maintainer="Nexedi",
+      maintainer_email="info@nexedi.com",
+      url="https://lab.nexedi.com/nexedi/slapos",
+      packages=find_packages(),
+      install_requires=[
+        'slapos.core',
+        'slapos.cookbook',
+        'slapos.libnetworkcache',
+        'supervisor',
+        'six',
+        'requests'
+        ],
+      zip_safe=True,
+      test_suite='test',
+    )

software/metabase/test/test.py

+##############################################################################
+# coding: utf-8
+#
+# Copyright (c) 2020 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import json
+from six.moves.urllib import parse
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, MetabaseTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestMetabaseSetup(MetabaseTestCase):
+  __partition_reference__ = 'S'  # postgresql use a socket in data dir
+  # instance can take time, /api/session/properties timeout at the beginning.
+  instance_max_retry = 30
+
+  def test_setup(self):
+    url = self.computer_partition.getConnectionParameterDict()['url']
+    resp = requests.get(parse.urljoin(url, '/setup'), verify=False)
+    self.assertTrue(resp.text)
+
+    # get a setup token as described in https://github.com/metabase/metabase/issues/4240#issuecomment-290717451
+    properties = requests.get(
+        parse.urljoin(url, '/api/session/properties'), verify=False,
+        timeout=10).json()
+
+    email = "youlooknicetoday@email.com"
+    password = "password123456"
+
+    request_json = {
+        'token': properties['setup-token'],
+        'prefs': {
+            'allow_tracking': 'false',
+            'site_name': 'Org'
+        },
+        'user': {
+            'email': email,
+            'password': password,
+            'first_name': 'Johnny',
+            'last_name': 'Appleseed',
+            'site_name': 'Org',
+        },
+        'database': None
+    }
+    resp = requests.post(
+        parse.urljoin(url, '/api/setup'),
+        json=request_json,
+        verify=False,
+        timeout=5)
+    self.assertTrue(resp.ok)
+
+    resp = requests.post(
+        parse.urljoin(url, '/api/session'),
+        verify=False,
+        json={
+            "username": email,
+            "password": "wrong"
+        })
+    self.assertEqual(requests.codes.bad_request, resp.status_code)
+
+    session = requests.post(
+        parse.urljoin(url, '/api/session'),
+        verify=False,
+        json={
+            "username": email,
+            "password": password
+        }).json()
+    self.assertTrue(session.get('id'))

software/slapos-sr-testing/software.cfg

@@ -98,6 +98,11 @@ setup = ${slapos-repository:location}/software/seleniumserver/test/
 egg = slapos.test.slaprunner
 setup = ${slapos-repository:location}/software/slaprunner/test/
 
+[slapos.test.metabase-setup]
+<= setup-develop-egg
+egg = slapos.test.metabase
+setup = ${slapos-repository:location}/software/metabase/test/
+
 [slapos.test.helloworld-setup]
 <= setup-develop-egg
 egg = slapos.test.helloworld
@@ -187,6 +192,7 @@ eggs =
   ${slapos.test.theia-setup:egg}
   ${slapos.test.cloudooo-setup:egg}
   ${slapos.test.dream-setup:egg}
+  ${slapos.test.metabase-setup:egg}
   ${backports.lzma:egg}
 entry-points =
   runTestSuite=erp5.util.testsuite:runTestSuite
@@ -229,6 +235,7 @@ context =
 tests =
   ${slapos.test.kvm-setup:setup}
   ${slapos.test.slaprunner-setup:setup}
+  ${slapos.test.metabase-setup:setup}
   ${:extra}
 extra =
   ${slapos.cookbook-setup:setup}