software/erp5: Add with-max-rlimit-nofile option

nexedi/erp5@39369169 added the functionality to ERP5 to
set its soft limit of allowed open file descriptors to the system wide hard limit.
This parameter is useful for Wendelin based instances where the 1024
limit is easily reached.

With this patch, this parameter can also be set via SlapOS, which
simplifies usage of the Wendelin SR.

/reviewed-by @jerome, @jm, @rafael, @vpelletier
/reviewed-on nexedi/slapos!1465

---

This patch was already added in nexedi/slapos!1451, but
removed (by force push), due to issues discussed in nexedi/slapos!1451 (comment 193296).

software/erp5/instance-erp5-input-schema.json

@@ -103,6 +103,11 @@
         "null"
       ]
     },
+    "with-max-rlimit-nofile": {
+      "description": "Set open file descriptors soft limit to hard limit",
+      "default": false,
+      "type": "boolean"
+    },
     "family-override": {
       "description": "Family-wide options, possibly overriding global options",
       "default": {},

software/erp5/test/test/test_erp5.py

@@ -32,6 +32,7 @@ import glob
 import http.client
 import json
 import os
+import resource
 import shutil
 import socket
 import sqlite3
@@ -1100,3 +1101,36 @@ class TestNEO(ZopeSkinsMixin, CrontabMixin, ERP5InstanceTestCase):
           'var',
           'log',
           f))
+
+
+class TestWithMaxRlimitNofileParameter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test setting the with-max-rlimit-nofile parameter sets the open fd soft limit to the hard limit.
+  """
+  __partition_reference__ = 'with-max-rlimit-nofile'
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {'_': json.dumps({'with-max-rlimit-nofile': True})}
+
+  def test_with_max_rlimit_nofile(self):
+    with self.slap.instance_supervisor_rpc as supervisor:
+      all_process_info = supervisor.getAllProcessInfo()
+    _, current_hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+    process_info, = (p for p in all_process_info if p['name'].startswith('zope-'))
+    self.assertEqual(
+      resource.prlimit(process_info['pid'], resource.RLIMIT_NOFILE),
+      (current_hard_limit, current_hard_limit))
+
+
+class TestUnsetWithMaxRlimitNofileParameter(ERP5InstanceTestCase, TestPublishedURLIsReachableMixin):
+  """Test not setting the with-max-rlimit-nofile parameter doesn't change the soft limit of erp5
+  """
+  __partition_reference__ = 'unset-with-max-rlimit-nofile'
+
+  def test_unset_with_max_rlimit_nofile(self):
+    with self.slap.instance_supervisor_rpc as supervisor:
+      all_process_info = supervisor.getAllProcessInfo()
+    limit = resource.getrlimit(resource.RLIMIT_NOFILE)
+    process_info, = (p for p in all_process_info if p['name'].startswith('zope-'))
+    self.assertEqual(
+      resource.prlimit(process_info['pid'], resource.RLIMIT_NOFILE), limit)

stack/erp5/buildout.hash.cfg

@@ -74,7 +74,7 @@ md5sum = 3f7b28085ceff321a3cb785db60f7c3e
 
 [template-erp5]
 filename = instance-erp5.cfg.in
-md5sum = 098e1d02159aeca9b36f2a0726b7b230
+md5sum = 84fb003691a7541835439c4e10be9953
 
 [template-zeo]
 filename = instance-zeo.cfg.in
@@ -86,7 +86,7 @@ md5sum = 0ac4b74436f554cd677f19275d18d880
 
 [template-zope]
 filename = instance-zope.cfg.in
-md5sum = 9e6440425167a506bd473a3697eaa9e6
+md5sum = 8a2c78000e1e791dc48305a2e7330b16
 
 [template-balancer]
 filename = instance-balancer.cfg.in

stack/erp5/instance-erp5.cfg.in

@@ -311,6 +311,7 @@ config-longrequest-logger-interval = {{ dumps(zope_parameter_dict.get('longreque
 config-longrequest-logger-timeout = {{ dumps(zope_parameter_dict.get('longrequest-logger-timeout', 1)) }}
 config-large-file-threshold = {{ dumps(zope_parameter_dict.get('large-file-threshold', "10MB")) }}
 config-port-base = {{ dumps(zope_parameter_dict.get('port-base', 2200)) }}
+config-with-max-rlimit-nofile = {{ dumps(slapparameter_dict.get('with-max-rlimit-nofile', false)) }}
 {# BBB: zope_parameter_dict used to contain 'webdav', so fallback to it -#}
 config-webdav = {{ dumps(current_zope_family_override_dict.get('webdav', zope_parameter_dict.get('webdav', False))) }}
 config-publisher-timeout = {{ dumps(current_zope_family_override_dict.get('publisher-timeout', global_publisher_timeout)) }}

stack/erp5/instance-zope.cfg.in

 {% from "instance_zodb_base" import zodb_dict  with context %}
 {% set webdav = slapparameter_dict['webdav'] -%}
+{% set with_max_rlimit_nofile = slapparameter_dict['with-max-rlimit-nofile'] -%}
 {% set thread_amount = slapparameter_dict['thread-amount'] %}
 {% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
 {% set ports = itertools.count(slapparameter_dict['port-base']) -%}
@@ -346,6 +347,7 @@ wrapped-command-line =
     --access-log-file={{ '${' ~ conf_parameter_name ~ ':z2-log}' }}
     {% if longrequest_logger_interval > 0 %} --long-request-log-file={{ '${' ~ conf_parameter_name ~ ':longrequest-logger-file}' }} {% endif %}
     {% if webdav %}-w{% endif %}
+    {% if with_max_rlimit_nofile %}--with-max-rlimit-nofile{% endif %}
     {{ ipv4 }}:${:port}
     {% if timerserver_interval %}--timerserver-interval={{ timerserver_interval }}{% endif %}
     '${:configuration-file}'