Move zope configuration file out of recipe, into a template.

slapos/recipe/generic_zope_zeo_client/__init__.py

@@ -43,7 +43,6 @@ def Zope2InitUser(path, username, password):
 class Recipe(GenericBaseRecipe):
   def _options(self, options):
     options['password'] = self.generatePassword()
-    options['deadlock-password'] = self.generatePassword()
 
   def install(self):
     """
@@ -91,25 +90,6 @@ class Recipe(GenericBaseRecipe):
         # Always provide a URL-Type
         append("file://" + link)
 
-    # Generate Zeo connections
-    zeo_snippet_template = open(self.getTemplateFilename('zope.zeo.entry.conf.in'
-      )).read()
-    zeo_snippet_list = []
-    for zeo_line in self.options['zeo-connection-string'].splitlines():
-      zeo_line.strip()
-      if not zeo_line:
-        continue
-      d = dict()
-      for param in zeo_line.split():
-        k, v = param.split('=')
-        d[k.strip()] = v.strip()
-      zeo_snippet_list.append(zeo_snippet_template % d)
-    # Create zope configuration file
-    zope_config = dict(
-        products=self.options['products'],
-        thread_amount=self.options['thread-amount'],
-        zodb_configuration='\n'.join(zeo_snippet_list)
-    )
     zope_environment = dict(
       TMP=self.options['tmp-path'],
       TMPDIR=self.options['tmp-path'],
@@ -134,44 +114,7 @@ class Recipe(GenericBaseRecipe):
     # configure default Zope2 zcml
     open(self.options['site-zcml'], 'w').write(open(self.getTemplateFilename(
         'site.zcml')).read())
-    zope_config['instance'] = self.options['instance-path']
-    zope_config['event_log'] = self.options['event-log']
-    zope_config['z2_log'] = self.options['z2-log']
-    zope_config['pid-filename'] = self.options['pid-file']
-    zope_config['lock-filename'] = self.options['lock-file']
-    prefixed_products = []
-    for product in reversed(zope_config['products'].split()):
-      product = product.strip()
-      if product:
-        prefixed_products.append('products %s' % product)
-    prefixed_products.insert(0, 'products %s' % self.options[
-      'instance-products'])
-    zope_config['products'] = '\n'.join(prefixed_products)
-    zope_config['address'] = '%s:%s' % (self.options['ip'], self.options['port'])
-    zope_config.update(dump_url=self.options['deadlock-path'],
-      secret=self.options['deadlock-password'])
-
-    zope_wrapper_template_location = self.getTemplateFilename('zope.conf.in')
-    zope_conf_content = self.substituteTemplate(zope_wrapper_template_location,
-      zope_config)
-    if self.isTrueValue(self.options['timeserver']):
-      zope_conf_content += self.substituteTemplate(self.getTemplateFilename(
-          'zope.conf.timeserver.in'), {})
-    if 'tidstorage-ip' in self.options:
-      zope_conf_content += self.substituteTemplate(self.getTemplateFilename(
-          'zope.conf.tidstorage.in'), {
-            'tidstorage-ip': self.options['tidstorage-ip'],
-            'tidstorage-port': self.options['tidstorage-port'],
-            })
-    if ('promise-path' in self.options) and ('site-id' in self.options):
-      zope_conf_content += self.substituteTemplate(self.getTemplateFilename(
-          'zope.conf.promise.in'), {
-            'site-id': self.options['site-id'],
-            'promise-path': self.options['promise-path'],
-            })
 
-    zope_conf_path = self.createFile(self.options['configuration-file'], zope_conf_content)
-    path_list.append(zope_conf_path)
     # Create init script
-    path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', zope_conf_path], zope_environment]))
+    path_list.append(self.createPythonScript(self.options['wrapper'], 'slapos.recipe.librecipe.execute.executee', [[self.options['runzope-binary'].strip(), '-C', self.options['configuration-file']], zope_environment]))
     return path_list

software/erp5/instance-tidstorage.cfg.in

@@ -39,29 +39,41 @@ backward compatibility with existing automatically setup CAs.
   longrequest_logger_timeout='',
   longrequest_logger_interval=''
 ) -%}
-[{{ section(name) }}]
-< = zope-base
+{% set conf_name = name ~ '-conf' -%}
+{% set conf_parameter_name = conf_name ~ '-param' %}
+[{{ conf_parameter_name }}]
+< = zope-conf-parameter-base
+pid-file = ${basedirectory:run}/{{ name }}.pid
+lock-file = ${basedirectory:run}/{{ name }}.lock
 {% do zope_dummy_list.append(None) %}
 {% set offset = zope_dummy_list | length %}
 port = {{ zope_port_base + offset }}
 thread-amount = {{ thread_amount }}
 timeserver = {{ timeserver }}
+event-log = ${basedirectory:log}/{{ name }}-event.log
+z2-log = ${basedirectory:log}/{{ name }}-Z2.log
+
+[{{ conf_name }}]
+< = zope-conf-base
+rendered = ${rootdirectory:etc}/{{ name }}.conf
+extra-context =
+  section parameter_dict {{ conf_parameter_name }}
+
+[{{ section(name) }}]
+< = zope-base
 longrequest-logger-file = {{ longrequest_logger_file }}
 longrequest-logger-timeout = {{ longrequest_logger_timeout }}
 longrequest-logger-interval = {{ longrequest_logger_interval }}
 wrapper = ${basedirectory:services}/{{ name }}
-pid-file = ${basedirectory:run}/{{ name }}.pid
-lock-file = ${basedirectory:run}/{{ name }}.lock
-event-log = ${basedirectory:log}/{{ name }}-event.log
-z2-log = ${basedirectory:log}/{{ name }}-Z2.log
-configuration-file = ${rootdirectory:etc}/{{ name }}.conf
+configuration-file = {{ '${' ~ conf_name ~ ':rendered}' }}
+port = {{ '${' ~ conf_parameter_name ~ ':port}' }}
 
 [{{ section('logrotate-entry-' ~ name) }}]
 < = logrotate-base
 recipe = slapos.cookbook:logrotate.d
 name = {{ name }}
-log = {{ '${' ~ name ~ ':event-log}' }} {{ '${' ~ name ~ ':z2-log}' }}
-post = {{ bin_directory }}/killpidfromfile {{ '${' ~ name ~ ':pid-file}' }} SIGUSR2
+log = {{ '${' ~ conf_parameter_name ~ ':event-log}' }} {{ '${' ~ conf_parameter_name ~ ':z2-log}' }}
+post = {{ bin_directory }}/killpidfromfile {{ '${' ~ conf_parameter_name ~ ':pid-file}' }} SIGUSR2
 {% endmacro -%}
 #############################
 # Directory creation
@@ -261,26 +273,28 @@ ip = {{ ipv4 }}
 binary-path = {{ bin_directory }}/runzeo
 
 {% for zeo_id, zeo_configuration_list in json['zeo'].iteritems() -%}
+{%   set current_zeo_port = current_zeo_port + 1 -%}
 {%   set storage_list = [] -%}
 {%   for zeo_slave in zeo_configuration_list -%}
-{%     do zodb_connection_list.append(
-      'zeo-cache-size=%(zeo-cache-size)s zope-cache-size=%(zope-cache-size)s '
-      'server=%(server)s mount-point=%(mount-point)s storage-name=%(storage-name)s' % {
-        'zope-cache-size': zeo_slave['zope-cache-size'],
-        'zeo-cache-size': zeo_slave['zeo-cache-size'],
-        'mount-point': zeo_slave['mount-point'] % {'site-id': site_id},
-        'storage-name': zeo_slave['storage-name'],
-        'server': '${zeo-instance-%(zeo-id)s:ip}:${zeo-instance-%(zeo-id)s:port}' % {'zeo-id': zeo_id},
-    }) -%}
+{%     do zodb_connection_list.append((
+         zeo_slave['storage-name'],
+         zeo_slave['mount-point'] % {'site-id': site_id},
+         zeo_slave['zope-cache-size'],
+         'zeoclient',
+         {
+           'cache-size': zeo_slave['zeo-cache-size'],
+           'server': ipv4 ~ ':' ~ current_zeo_port,
+           'storage': zeo_slave['storage-name'],
+           'name': zeo_slave['storage-name'],
+         },
+       )) -%}
 {%     set zodb_path = '${directory:zodb}/' ~ zeo_slave['storage-name'] ~ '.fs' -%}
 {%     do storage_list.append('storage-name=%(storage-name)s zodb-path=%(zodb-path)s' % {'zodb-path': zodb_path, 'storage-name': zeo_slave['storage-name']}) -%}
-{%     do known_tid_storage_identifier_dict.__setitem__("((('%(ip)s', %(port)s),), '%(storage_name)s')" % {
-        'ip': '${zeo-instance-%s:ip}' % zeo_id,
-        'port': '${zeo-instance-%s:port}' % zeo_id,
+{%     do known_tid_storage_identifier_dict.__setitem__("((('${zeo-instance-%(zeo_id)s:ip}', ${zeo-instance-%(zeo_id)s:port}),), '%(storage_name)s')" % {
+         'zeo_id': zeo_id,
          'storage_name': zeo_slave['storage-name']
        }, (zodb_path, '${directory:zodb-backup}/%s/' % zeo_slave['storage-name'], zeo_slave['serialize-path'] % {'site-id': site_id})) -%}
 {%   endfor -%}
-{%   set current_zeo_port = current_zeo_port + 1 -%}
 [{{ section('zeo-instance-%s' % zeo_id) }}]
 < = zeo-instance-entry-common
 log-path = ${basedirectory:log}/zeo-{{ zeo_id }}.log
@@ -299,36 +313,52 @@ log = ${zeo-instance-{{ zeo_id }}:log-path}
 post = {{ bin_directory }}/killpidfromfile ${zeo-instance-{{ zeo_id }}:pid-path} SIGUSR2
 
 {% endfor -%}
+#############################
+# Zope
+#############################
 [zope-base]
 recipe = slapos.cookbook:generic.zope.zeo.client
-# XXX: products won't be needed as soon as all ERP5 (and products-deps)
-# products will be eggified so then it will be possible to use them thanks to
-# availability in software's eggs
-products = {{ parameter_dict['products'] }}
 user = zope
 ip = {{ ipv4 }}
-deadlock-path = /manage_debug_threads
 timezone = {{ json['timezone'] }}
-zeo-connection-string =
-  {{ zodb_connection_list | join('\n  ') }}
 tidstorage-ip = ${tidstorage:ip}
 tidstorage-port = ${tidstorage:port}
-instance-path = ${directory:instance}
 instance-etc = ${directory:instance-etc}
-instance-products = ${directory:instance-products}
 bt5-repository = ${rootdirectory:var}/bt5_repository
 tmp-path = ${rootdirectory:tmp}
 bin-path = ${rootdirectory:bin}
 site-zcml = ${:instance-etc}/site.zcml
-inituser = ${:instance-path}/inituser
+inituser = ${directory:instance}/inituser
 runzope-binary = {{ bin_directory }}/runzope
 bt5-repository-list =
-promise-path = ${erp5-promise:promise-path}
+
+[deadlock-debugger-password]
+recipe = slapos.cookbook:pwgen.stable
+
+[zope-conf-parameter-base]
+ip = {{ ipv4 }}
 site-id = {{ site_id }}
+zodb-list = {{ json_module.dumps(zodb_connection_list) }}
+# XXX: products won't be needed as soon as all ERP5 (and products-deps)
+# products will be eggified so then it will be possible to use them thanks to
+# availability in software's eggs
+products = {{ parameter_dict['products'] }}
+
+[zope-conf-base]
+recipe = slapos.recipe.template:jinja2
+template = {{ parameter_dict['zope-conf-template'] }}
+extra-context =
+context =
+  key instance directory:instance
+  key instance_products directory:instance-products
+  raw deadlock_path /manage_debug_threads
+  key deadlock_debugger_password deadlock-debugger-password:password
+  import json_module json
+  key tidstorage_ip tidstorage:ip
+  key tidstorage_port tidstorage:port
+  key promise_path erp5-promise:promise-path
+  ${:extra-context}
 
-#############################
-# Zope
-#############################
 # Distribution node
 {{ zope('zope-distribution', timeserver=True) }}
 # Admin node

software/erp5/instance.cfg.in

@@ -48,6 +48,7 @@ aspell = {{ aspell_location }}
 poppler = {{ poppler_location }}
 dmtx-utils = {{ dmtx_utils_location }}
 buildout-bin-directory = {{ buildout_bin_directory }}
+zope-conf-template = {{ template_zope_conf }}
 
 [dynamic-template-tidstorage]
 < = jinja2-template-base

software/erp5/software.cfg

@@ -132,7 +132,7 @@ extra-context =
 [template-tidstorage]
 recipe = slapos.recipe.build:download
 url = ${:_profile_base_location_}/instance-tidstorage.cfg.in
-md5sum = 520cf89a9b495a474519ac3402fed170
+md5sum = 575185cf6693244b8c103aa9cdcd6f91
 mode = 640
 
 [template-cloudooo]
@@ -144,12 +144,18 @@ mode = 640
 # Additional Configuration
 configurator_bt5_list = erp5_core_proxy_field_legacy erp5_full_text_myisam_catalog erp5_base erp5_workflow erp5_configurator erp5_configurator_standard erp5_configurator_maxma_demo erp5_configurator_ung
 
+[template-zope-conf]
+recipe = slapos.recipe.build:download
+url = ${:_profile_base_location_}/zope.conf.in
+md5sum = 5707c4d8771a53382f1b68fa63f93c80
+mode = 640
+
 [template]
 < = template-jinja2-base
 # XXX: "template.cfg" is hardcoded in instanciation recipe
 filename = template.cfg
 template = ${:_profile_base_location_}/instance.cfg.in
-md5sum = f776aaa2c1b686841acf4c8b8192a782
+md5sum = d9b0d9105a4a55e6fca32e5b550d12f2
 extra-context =
     key apache_location apache:location
     key aspell_location aspell:location
@@ -195,6 +201,7 @@ extra-context =
     key template_tidstorage template-tidstorage:target
     key template_varnish template-varnish:target
     key template_zope template-zope:rendered
+    key template_zope_conf template-zope-conf:target
     key tesseract_location tesseract:location
     key w3m_location w3m:location
     key zlib_location zlib:location

software/erp5/zope.conf.in

+# Note: Environment is setup in running wrapper script, as zope.conf is read
+# too late for some components.
+%define INSTANCE {{ instance }}
+instancehome $INSTANCE
+zserver-threads {{ parameter_dict['thread-amount'] }}
+
+lock-filename {{ parameter_dict['lock-file'] }}
+pid-filename {{ parameter_dict['pid-file'] }}
+
+default-zpublisher-encoding utf-8
+rest-input-encoding utf-8
+rest-output-encoding utf-8
+
+# XXX: isn't this entry implicit ?
+products {{ instance_products }}
+{% for product in parameter_dict['products'].split() -%}
+{%   if product.strip() -%}
+products {{ product }}
+{%   endif -%}
+{% endfor -%}
+
+<http-server>
+  address {{ parameter_dict['ip'] }}:{{ parameter_dict['port'] }}
+</http-server>
+
+{# TODO: enable
+{% if webdav_port -%}
+<webdav-source-server>
+  address {{ parameter_dict['ip'] }}:{{ parameter_dict['webdav_port'] }}
+  force-connection-close off
+</webdav-source-server>
+
+{% endif -%}
+-#}
+<zoperunner>
+    program $INSTANCE/bin/runzope
+</zoperunner>
+
+<product-config DeadlockDebugger>
+  dump_url /manage_debug_threads
+  secret {{ deadlock_debugger_password }}
+</product-config>
+
+{% if tidstorage_ip -%}
+<product-config TIDStorage>
+  backend-ip {{ tidstorage_ip }}
+  backend-port {{ tidstorage_port }}
+</product-config>
+
+{% endif -%}
+{# BBB: should not have to compare those values to begin with: just take a
+parameter with timerserver interval, empty string meaning disabled -#}
+{% if parameter_dict['timeserver'] in ('y', 'yes', '1', 'true') -%}
+%import timerserver
+<timer-server>
+  interval 1
+</timer-server>
+
+{% endif -%}
+{% if promise_path -%}
+<product-config /{{ parameter_dict['site-id'] }}>
+  promise_path {{ promise_path }}
+</product-config>
+
+{% endif -%}
+<eventlog>
+  level info
+  <logfile>
+    dateformat
+    path {{ parameter_dict['event-log'] }}
+  </logfile>
+</eventlog>
+
+<logger access>
+  level WARN
+  <logfile>
+    dateformat
+    path {{ parameter_dict['z2-log'] }}
+  </logfile>
+</logger>
+
+<zodb_db temporary>
+  <temporarystorage>
+    name temporary storage for sessioning
+  </temporarystorage>
+  mount-point /temp_folder
+  container-class Products.TemporaryFolder.TemporaryContainer
+</zodb_db>
+
+{% for db_name, mount_point, cache_size, storage_type, storage_dict in parameter_dict['zodb-list'] -%}
+<zodb_db {{ db_name }}>
+{%-   if cache_size %}
+  cache-size {{ cache_size }}
+{%-   endif %}
+  mount-point {{ mount_point }}
+  <{{ storage_type }}>
+{%-   for key, value in storage_dict.iteritems() %}
+    {{ key }} {{ value }}
+{%-   endfor %}
+  </{{ storage_type }}>
+</zodb_db>
+
+{% endfor -%}