Update Release Candidate

software/erp5/test/test/__init__.py

@@ -179,6 +179,10 @@ class ERP5InstanceTestCase(SlapOSInstanceTestCase, metaclass=ERP5InstanceTestMet
   """
   __test_matrix__ = matrix((zeo, neo))  # switch between NEO and ZEO mode
 
+  @classmethod
+  def isNEO(cls):
+    return '_neo' in cls.__name__
+
   @classmethod
   def getRootPartitionConnectionParameterDict(cls):
     """Return the output parameters from the root partition"""

software/erp5/test/test/test_erp5.py

@@ -34,6 +34,7 @@ import json
 import os
 import shutil
 import socket
+import sqlite3
 import ssl
 import subprocess
 import sys
@@ -48,7 +49,7 @@ import xmlrpc.client
 import urllib3
 from slapos.testing.utils import CrontabMixin
 
-from . import ERP5InstanceTestCase, setUpModule, matrix, default
+from . import ERP5InstanceTestCase, setUpModule, matrix, default, neo
 
 setUpModule # pyflakes
 
@@ -789,6 +790,39 @@ class ZopeTestMixin(ZopeSkinsMixin, CrontabMixin):
     self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
     self.assertFalse(os.path.exists(rotated_log_file))
 
+  def test_neo_root_log_rotation(self):
+    zope_neo_root_log_path = os.path.join(
+      self.getComputerPartitionPath('zope-default'),
+      'var',
+      'log',
+      'zope-0-neo-root.log',
+    )
+    if not self.isNEO():
+      self.assertFalse(os.path.exists(zope_neo_root_log_path))
+      return
+
+    def check_sqlite_log(path):
+      with contextlib.closing(sqlite3.connect(path)) as con:
+        con.execute('select * from log')
+
+    check_sqlite_log(zope_neo_root_log_path)
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+
+    rotated_log_file = os.path.join(
+      self.getComputerPartitionPath('zope-default'),
+      'srv',
+      'backup',
+      'logrotate',
+      'zope-0-neo-root.log-20500101',
+    )
+    check_sqlite_log(rotated_log_file)
+
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
+    self.assertFalse(os.path.exists(rotated_log_file))
+    requests.get(self._getAuthenticatedZopeUrl('/'), verify=False).raise_for_status()
+    check_sqlite_log(zope_neo_root_log_path)
+
   def test_basic_authentication_user_in_access_log(self):
     param_dict = self.getRootPartitionConnectionParameterDict()
     requests.get(self.zope_base_url,
@@ -866,7 +900,7 @@ class ZopeTestMixin(ZopeSkinsMixin, CrontabMixin):
         'zope-2-Z2.log',
         'zope-2-event.log',
         'zope-2-neo-root.log',
-      ] if '_neo' in self.__class__.__name__ else [
+      ] if self.isNEO() else [
         'zope-0-Z2.log',
         'zope-0-event.log',
         'zope-1-Z2.log',
@@ -1004,3 +1038,66 @@ class TestCloudoooDefaultParameter(ZopeSkinsMixin, ERP5InstanceTestCase):
             'portal_preferences/getPreferredDocumentConversionServerRetry'),
           verify=False).text,
       "2")
+
+
+class TestNEO(ZopeSkinsMixin, CrontabMixin, ERP5InstanceTestCase):
+  """Tests specific to neo storage
+  """
+  __partition_reference__ = 'n'
+  __test_matrix__ = matrix((neo,))
+
+  def _getCrontabCommand(self, crontab_name):
+    # type: (str) -> str
+    """Read a crontab and return the command that is executed.
+
+    overloaded to use crontab from neo partition
+    """
+    with open(
+        os.path.join(
+            self.getComputerPartitionPath('neo-0'),
+            'etc',
+            'cron.d',
+            crontab_name,
+        )) as f:
+      crontab_spec, = f.readlines()
+    self.assertNotEqual(crontab_spec[0], '@', crontab_spec)
+    return crontab_spec.split(None, 5)[-1]
+
+  def test_log_rotation(self):
+    # first run to create state files
+    self._executeCrontabAtDate('logrotate', '2000-01-01')
+
+    def check_sqlite_log(path):
+      with self.subTest(path), contextlib.closing(sqlite3.connect(path)) as con:
+        con.execute('select * from log')
+
+    logfiles = ('neoadmin.log', 'neomaster.log', 'neostorage-0.log')
+    for f in logfiles:
+      check_sqlite_log(
+        os.path.join(
+          self.getComputerPartitionPath('neo-0'),
+          'var',
+          'log',
+          f))
+
+    self._executeCrontabAtDate('logrotate', '2050-01-01')
+
+    for f in logfiles:
+      check_sqlite_log(
+        os.path.join(
+          self.getComputerPartitionPath('neo-0'),
+          'srv',
+          'backup',
+          'logrotate',
+          f'{f}-20500101'))
+
+    self._executeCrontabAtDate('logrotate', '2050-01-02')
+    requests.get(self._getAuthenticatedZopeUrl('/'), verify=False).raise_for_status()
+
+    for f in logfiles:
+      check_sqlite_log(
+        os.path.join(
+          self.getComputerPartitionPath('neo-0'),
+          'var',
+          'log',
+          f))

software/galene/instance-default-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by galene",
+  "properties": {
+    "url": {
+      "description": "IPv6 url of galene",
+      "type": "string"
+    },
+    "admin-user": {
+      "description": "Admin username",
+      "type": "string"
+    },
+    "admin-password": {
+      "description": "Admin password",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/galene/instance-ptt-default-input-schema.json

@@ -16,10 +16,10 @@
       "default": true
     },
     "dns_sr_url": {
-     "default": "",
-     "title": "DNS SR URL",
-     "description": "URL of the SR running the DNS server",
-     "type": "string"
+      "default": "",
+      "title": "DNS SR URL",
+      "description": "URL of the SR running the DNS server",
+      "type": "string"
     }
   }
 }

software/galene/instance-ptt-default-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by galene",
+  "properties": {
+    "url": {
+      "description": "IPv6 url of galene",
+      "type": "string"
+    },
+    "admin-user": {
+      "description": "Admin username",
+      "type": "string"
+    },
+    "admin-password": {
+      "description": "Admin password",
+      "type": "string"
+    },
+    "domain-url": {
+      "description": "URL of galene using domain configured with local DNS",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/galene/software-ptt.cfg.json

@@ -8,7 +8,7 @@
       "software-type": "default",
       "description": "default",
       "request": "instance-ptt-default-input-schema.json",
-      "response": "instance-default-schema.json",
+      "response": "instance-ptt-default-output-schema.json",
       "index": 0
     }
   }

software/galene/software.cfg.json

@@ -8,7 +8,7 @@
       "software-type": "default",
       "description": "default",
       "request": "instance-default-input-schema.json",
-      "response": "instance-default-schema.json",
+      "response": "instance-default-output-schema.json",
       "index": 0
     }
   }

software/mail-server/instance-default-input-schema.json

@@ -11,4 +11,3 @@
     }
   }
 }
-

software/mail-server/test/README.md

-Tests for mail-server software release

software/mail-server/test/setup.py

-##############################################################################
-#
-# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-from setuptools import setup, find_packages
-
-version = '0.0.1.dev0'
-name = 'slapos.test.mail_server'
-with open("README.md") as f:
-  long_description = f.read()
-
-setup(
-    name=name,
-    version=version,
-    description="Test for SlapOS' mail-server",
-    long_description=long_description,
-    long_description_content_type='text/markdown',
-    maintainer="Nexedi",
-    maintainer_email="info@nexedi.com",
-    url="https://lab.nexedi.com/nexedi/slapos",
-    packages=find_packages(),
-    install_requires=[
-        'slapos.core',
-        'slapos.libnetworkcache',
-        'slapos.cookbook',
-    ],
-    zip_safe=True,
-    test_suite='test',
-)

software/mail-server/test/test.py

-##############################################################################
-#
-# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
-#
-# WARNING: This program as such is intended to be used by professional
-# programmers who take the whole responsibility of assessing all potential
-# consequences resulting from its eventual inadequacies and bugs
-# End users who are looking for a ready-to-use solution with commercial
-# guarantees and support are strongly adviced to contract a Free Software
-# Service Company
-#
-# This program is Free Software; you can redistribute it and/or
-# modify it under the terms of the GNU General Public License
-# as published by the Free Software Foundation; either version 3
-# of the License, or (at your option) any later version.
-#
-# This program is distributed in the hope that it will be useful,
-# but WITHOUT ANY WARRANTY; without even the implied warranty of
-# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
-# GNU General Public License for more details.
-#
-# You should have received a copy of the GNU General Public License
-# along with this program; if not, write to the Free Software
-# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
-#
-##############################################################################
-
-import os
-
-from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
-
-setUpModule, MailServerTestCase = makeModuleSetUpAndTestCaseClass(
-    os.path.abspath(
-        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
-
-param_dict = {
-    "mail_domain": "mail.local",
-}
-
-class TestDefaultInstance(MailServerTestCase):
-    @classmethod
-    def getInstanceParameterDict(cls):
-        return {'_': json.dumps(param_dict)}
-    @classmethod
-    def getInstanceSoftwareType(cls):
-        return "default"
-    def test_enb_conf(self):
-      self.slap.waitForInstance()
-      connection_parameters = self.computer_partition.getConnectionParameterDict()
-
-      imap_smtp_ipv6 = connection_parameters['imap-smtp-ipv6']
-      imap_port = connection_parameters['imap-port']
-      smtp_port = connection_parameters['smtp-port']
-
-      # Check connection parameters are not empty
-      self.assertTrue(imap_smtp_ipv6)
-      self.assertTrue(imap_port)
-      self.assertTrue(smtp_port)
-
-      # Check conf contains correct domain
-      conf_file = glob.glob(os.path.join(
-        self.slap.instance_directory, '*', 'etc', 'postfix', 'main.cf'))[0]
-
-      with open(conf_file, 'r') as f:
-        domain_configured = False
-        for line in f:
-          if line.startswith("virtual_mailbox_domains"):
-            self.assertEqual(line, "virtual_mailbox_domains = {}\n".format(param_dict['mail_domain']))
-            domain_configured = True
-        self.assertTrue(domain_configured)

software/monitor/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
 
 [template-monitor-edgetest-basic]
 _update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
-md5sum = fa044accc1230ece41edfc822eb39f07
+md5sum = d375f656087bfbd8a11188721e31de68
 
 [template-node-monitoring]
 _update_hash_filename_ = instance-node-monitoring.jinja2.cfg
@@ -38,4 +38,4 @@ md5sum = d3cfa1f6760e3fa64ccd64acf213bdfb
 
 [template-surykatka-ini]
 _update_hash_filename_ = surykatka.ini.jinja2
-md5sum = 7e9b874c20faaa8190d2bf2b74caa727
+md5sum = 4dbc223cb7294eecb31d19f3afb82e86

software/monitor/instance-monitor-edgetest-basic.cfg.jinja2

@@ -92,6 +92,7 @@ nameserver_list = {{ dumps(DEFAULT_DICT['nameserver-list']) }}
 json = ${directory:srv}/surykatka-{{ class }}.json
 {#- timeout is just a bit bigger than class time #}
 timeout = {{ int(class) + 2 }}
+check_maximum_elapsed_time = {{ int(class) }}
 
 context =
   import json_module json
@@ -99,6 +100,7 @@ context =
   key nameserver_list :nameserver_list
   key url_list :url_list
   key timeout :timeout
+  key check_maximum_elapsed_time :check_maximum_elapsed_time
 
 {%-     do PART_LIST.append('surykatka-%i'% (class,)) %}
 [surykatka-{{ class }}]

software/monitor/surykatka.ini.jinja2

@@ -2,6 +2,7 @@
 INTERVAL = 120
 TIMEOUT = {{ timeout }}
 SQLITE = {{ db }}
+ELAPSED_FAST = {{ check_maximum_elapsed_time }}
 {%- if len(nameserver_list) > 0 %}
 NAMESERVER =
 {%-   for nameserver_entry in sorted(nameserver_list) %}

software/monitor/test/test.py

@@ -344,6 +344,7 @@ class TestEdgeBasic(EdgeMixin, SlapOSInstanceTestCase):
 INTERVAL = 120
 TIMEOUT = 7
 SQLITE = %(db_file)s
+ELAPSED_FAST = 5
 NAMESERVER =
   127.0.1.1
   127.0.1.2
@@ -363,6 +364,7 @@ URL =
 INTERVAL = 120
 TIMEOUT = 13
 SQLITE = %(db_file)s
+ELAPSED_FAST = 11
 NAMESERVER =
   127.0.1.1
   127.0.1.2

software/slapos-sr-testing/software.cfg

@@ -262,11 +262,6 @@ setup = ${slapos-repository:location}/software/js-drone/test/
 egg = slapos.test.osie_coupler
 setup = ${slapos-repository:location}/software/osie-coupler/test/
 
-[slapos.test.mail-server-setup]
-<= setup-develop-egg
-egg = slapos.test.mail-server
-setup = ${slapos-repository:location}/software/mail-server/test/
-
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -369,7 +364,6 @@ eggs +=
   ${slapos.test.theia-setup:egg}
   ${slapos.test.turnserver-setup:egg}
   ${slapos.test.upgrade_erp5-setup:egg}
-  ${slapos.test.mail-server-setup:egg}
 
 # We don't name this interpreter `python`, so that when we run slapos node
 # software, installation scripts running `python` use a python without any
@@ -463,7 +457,6 @@ tests =
   theia ${slapos.test.theia-setup:setup}
   turnserver ${slapos.test.turnserver-setup:setup}
   upgrade_erp5 ${slapos.test.upgrade_erp5-setup:setup}
-  mail-server ${slapos.test.mail-server-setup:setup}
 
 [versions]
 # recurls are under development

stack/erp5/buildout.hash.cfg

@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 558ffbc6d51bb0ce9fc25d1062edcd2a
+md5sum = e6c94c2a48788683bf0d63d135a44932
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-zope.cfg.in

@@ -308,14 +308,14 @@ port = {{ port }}
 event-log = ${directory:log}/{{ name }}-event.log
 z2-log = ${directory:log}/{{ name }}-Z2.log
 node-id = {{ dumps(node_id_base ~ (node_id_index_format % index)) }}
-{% set log_list = [] -%}
+{% set neo_log_list = [] -%}
 {% set import_set = set() -%}
 {% for db_name, zodb in six.iteritems(zodb_dict) -%}
 {%   do zodb.setdefault('pool-size', thread_amount) -%}
 {%   if zodb['type'] == 'neo' -%}
 {%     do import_set.add('neo.client') -%}
 {%     set log = name ~ '-neo-' ~ db_name ~ '.log' -%}
-{%     do log_list.append('${directory:log}/' + log) -%}
+{%     do neo_log_list.append('${directory:log}/' + log) -%}
 {%     do zodb['storage-dict'].update(logfile='~/var/log/'+log) -%}
 {%   endif -%}
 {% endfor -%}
@@ -350,6 +350,7 @@ wrapped-command-line =
     '${:configuration-file}'
     --threads={{ thread_amount }}
     --large-file-threshold={{ slapparameter_dict['large-file-threshold'] }}
+    --pidfile={{ '${' ~ conf_parameter_name ~ ':pid-file}' }}
 {%- set private_dev_shm = slapparameter_dict['private-dev-shm'] %}
 {%- if private_dev_shm %}
 private-tmpfs = {{ private_dev_shm }} /dev/shm
@@ -408,8 +409,18 @@ config-maximum-delay = {{ slapparameter_dict["zope-longrequest-logger-maximum-de
 [{{ section('logrotate-entry-' ~ name) }}]
 < = logrotate-entry-base
 name = {{ name }}
-log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ conf_parameter_name ~ ':longrequest-logger-file}' }} {{ ' '.join(log_list) }}
+log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }} {{ '${' ~ conf_parameter_name ~ ':longrequest-logger-file}' }}
 copytruncate = true
+
+{% if neo_log_list -%}
+[{{ section('logrotate-entry-neo-' ~ name) }}]
+< = logrotate-entry-base
+name = neo-{{ name }}
+log = {{ ' '.join(neo_log_list) }}
+# we don't use copytruncate on neo logs, they are not regular text files but sqlite databases
+copytruncate =
+post = test ! -s {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} || {{ bin_directory }}/slapos-kill --pidfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} -s USR2
+{% endif %}
 {% endmacro -%}
 
 {% for i in instance_index_list -%}

stack/logrotate/buildout.hash.cfg

@@ -22,4 +22,4 @@ md5sum = 02c1009f8e0dc371cfc1290afef72ec7
 
 [template-logrotate-base]
 filename = instance-logrotate-base.cfg.in
-md5sum = 4e2baa1edd1d27831dda984769102a7c
+md5sum = 303fad78d62d6e29c0c547a9f64fa822

stack/logrotate/instance-logrotate-base.cfg.in

@@ -47,6 +47,8 @@ context =
 # - "post" with commands to execute after rotation
 # - "pre" with commands to execute before rotation
 # - "backup" with directory where to store logs
+# - "copytruncate" to use logrotate's copytruncate option, setting to ""
+#    (the default) disable copytruncate, setting to anything else enable copytruncate
 recipe = slapos.recipe.template:jinja2
 url = {{ logrotate_entry_template }}
 output = ${logrotate-conf-parameter:logrotate-entries}/${:name}
@@ -60,7 +62,7 @@ context =
   key rotate_num :rotate-num
   key nocompress :nocompress
   key delaycompress :delaycompress
-copytruncate = false
+copytruncate =
 post =
 pre =
 frequency = daily