{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%} {% set ipv4 = (ipv4_set | list)[0] -%} {% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%} {% set part_list = [] -%} {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%} {% set storage_dict = {} -%} {% set default_zodb_path = buildout_directory ~ '/srv/zodb' -%} {% set default_backup_path = buildout_directory ~ '/srv/backup/zodb' -%} {% set bin_directory = parameter_dict['buildout-bin-directory'] -%} [zeo-base] recipe = slapos.cookbook:zeo log-path = ${directory:log}/${:base-name}.log pid-path = ${directory:run}/${:base-name}.pid conf-path = ${directory:etc}/${:base-name}.conf wrapper-path = ${directory:services}/${:base-name} binary-path = {{ bin_directory }}/runzeo ip = {{ ipv4 }} {% set known_tid_storage_identifier_dict = {} -%} {% set zodb_dict = {} -%} {% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%} {% do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%} {% endfor -%} {% set tidstorage_port = slapparameter_dict['tcpv4-port'] + len(zodb_dict) -%} {% for family, zodb in zodb_dict.iteritems() -%} {% set storage_list = [] -%} {% set current_port = next_port() -%} {% set known_tid_storage_identifier_host = (ipv4, current_port), -%} {% for name, zodb in zodb -%} {% do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ current_port, 'storage': name}) %} {% set path = zodb.get('path', '%(zodb)s/%(name)s.fs') % {'zodb': default_zodb_path, 'name': name} -%} {% do storage_list.append((name, path)) -%} {% set backup_directory = zodb.get('backup', '%(backup)s/%(name)s') % {'backup': default_backup_path, 'name': name} -%} {# BBB: No mount-point specified because they're meaningless for ZEO and TIDStorage. Pass '' for compatibility, and not None because this would disable TIDStorage bootstrapping. -#} {% do known_tid_storage_identifier_dict.__setitem__(json_module.dumps( (known_tid_storage_identifier_host, name)), (path, backup_directory, '')) -%} {% endfor -%} {% set zeo_section_name = 'zeo-' ~ family %} [{{ zeo_section_name }}] < = zeo-base base-name = zeo-{{ family }} port = {{ current_port }} storage = {{ dumps(storage_list) }} [{{ section("logrotate-" ~ zeo_section_name) }}] < = logrotate-entry-base name = {{ "${" ~ zeo_section_name ~ ":base-name}" }} log = {{ "${" ~ zeo_section_name ~ ":log-path}" }} post = test ! -s {{ "${" ~ zeo_section_name ~":pid-path}" }} || {{ bin_directory }}/slapos-kill --pidfile {{ "${" ~ zeo_section_name ~ ":pid-path}" }} -s USR2 [{{ section(zeo_section_name ~ "-promise") }}] recipe = slapos.cookbook:check_port_listening hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }} port = {{ "${" ~ zeo_section_name ~ ":port}" }} path = ${directory:promises}/zeo-{{ family }} {% endfor -%} {% if backup_periodicity == 'never' -%} {% set known_tid_storage_identifier_dict = () %} {% set tidstorage_repozo_path = '' -%} {% else -%} [tidstorage] recipe = slapos.cookbook:tidstorage known-tid-storage-identifier-dict = {{ dumps(known_tid_storage_identifier_dict) }} configuration-path = ${directory:etc}/tidstorage.py ip = {{ ipv4 }} port = {{ tidstorage_port }} {% set tidstorage_repozo_path = slapparameter_dict.get('tidstorage-repozo-path', buildout_directory ~ '/srv/backup/tidstorage') -%} timestamp-file-path = {{ tidstorage_repozo_path ~ '/repozo_tidstorage_timestamp.log' }} {# BBB: recipe requires logfile-name for nothing because tidstorage runs in foreground mode -#} logfile-name = pidfile-name = ${directory:run}/tidstorage.pid {# TODO: Add support for backup status file, so that the status file can be close to the ZODB (rather than close to the backup files). And do it efficiently, to not copy the whole status file every time. -#} status-file = {{ tidstorage_repozo_path ~ '/tidstorage.tid' }} tidstorage-repozo-binary = {{ bin_directory }}/tidstorage_repozo tidstoraged-binary = {{ bin_directory }}/tidstoraged repozo-binary = {{ bin_directory }}/repozo repozo-wrapper = ${buildout:bin-directory}/tidstorage-repozo {% if len(known_tid_storage_identifier_dict) > 1 -%} tidstorage-wrapper = ${directory:services}/tidstoraged [{{ section("promise-tidstorage") }}] recipe = slapos.cookbook:check_port_listening hostname = ${tidstorage:ip} port = ${tidstorage:port} path = ${directory:promises}/tidstorage {% endif -%} [{{ section("cron-entry-tidstorage-backup") }}] # TODO: # - configurable full/incremental # - configurable retention recipe = slapos.cookbook:cron.d cron-entries = ${cron:cron-entries} name = tidstorage time = {{ dumps(backup_periodicity) }} command = ${tidstorage:repozo-wrapper} # Used for ERP5 resiliency or (more probably) # webrunner resiliency with erp5 inside. [{{ section("resiliency-exclude-file") }}] # Generate rdiff exclude file recipe = slapos.recipe.template:jinja2 mode = 644 template = {{ 'inline:{{ "${directory:zodb}/**\\n${directory:log}\\n" }}' }} rendered = ${directory:srv}/exporter.exclude [{{ section("resiliency-identity-signature-script")}}] # Generate identity script used by webrunner to check data integrity # It excludes repozo files as they already include a hash function # used to check backups when rebuilding the datafs recipe = slapos.cookbook:wrapper command-line = {{ bin_directory }}/backup-identity-script-excluding-path --exclude-path "srv/backup/logrotate" --exclude-path "srv/backup/zodb" wrapper-path = ${directory:srv}/.backup_identity_script mode = 770 [{{ section("resiliency-after-import-script") }}] # Generate after import script used by importer instance of webrunner recipe = collective.recipe.template input = inline: #!/bin/sh # DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE # OR ZODB DATA WILL BE ERASED. # This script will restore the repozo backup to the real # zodb location. It is launched by the clone (importer) instance of webrunner # in the end of the import script. # Depending on the output, it will create a file containing # the status of the restoration (success or failure). zodb_directory="${directory:zodb}" zodb_backup_directory="{{ default_backup_path }}" repozo="${tidstorage:repozo-binary}" EXIT_CODE=0 {% for family, zodb in zodb_dict.iteritems() -%} {% for name, zodb in zodb -%} {% set zeo_section_name = 'zeo-' ~ family %} storage_name="{{ name }}" zodb_path="$storage_name.fs" pid_file={{ "${" ~ zeo_section_name ~ ":pid-path}" }} if [ -e "$pid_file" ]; then pid=$(cat $pid_file) > /dev/null 2>&1 if kill -0 "$pid"; then echo "Zeo is already running with pid $pid. Aborting." exit 1 fi fi echo "Removing $zodb_path..." echo "Restoring $storage_name into $zodb_path..." $repozo --verify --repository="$zodb_backup_directory/$storage_name" CURRENT_EXIT_CODE=$? if [ ! "$CURRENT_EXIT_CODE"="0" ]; then EXIT_CODE="$CURRENT_EXIT_CODE" echo "$storage_name Backup verification failed. Backup data is inconsistent." fi $repozo --recover --output="$zodb_directory/$zodb_path" --repository="$zodb_backup_directory/$storage_name" CURRENT_EXIT_CODE=$? if [ ! "$CURRENT_EXIT_CODE"="0" ]; then EXIT_CODE="$CURRENT_EXIT_CODE" echo "$storage_name Backup restoration failed." fi {% endfor -%} {% endfor -%} exit $EXIT_CODE output = ${directory:srv}/runner-import-restore mode = 755 {% endif -%} [{{ section('promise-check-computer-memory') }}] recipe = slapos.cookbook:wrapper wrapper-path = ${directory:promises}/check-computer-memory command-line = "{{ parameter_dict["check-computer-memory-binary"] }}" -db ${monitor-instance-parameter:collector-db} --threshold "{{ slapparameter_dict["computer-memory-percent-threshold"] }}" --unit percent [publish] recipe = slapos.cookbook:publish.serialised storage-dict = {{ dumps(storage_dict) }} {% if len(known_tid_storage_identifier_dict) > 1 -%} tidstorage-ip = ${tidstorage:ip} tidstorage-port = ${tidstorage:port} {% else -%} tidstorage-ip = tidstorage-port = {% endif -%} monitor-base-url = ${monitor-publish-parameters:monitor-base-url} [directory] recipe = slapos.cookbook:mkdirectory bin = ${buildout:directory}/bin etc = ${buildout:directory}/etc services = ${:etc}/run promises = ${:etc}/promise srv = ${buildout:directory}/srv var = ${buildout:directory}/var log = ${:var}/log run = ${:var}/run backup-zodb = {{ default_backup_path }} zodb = {{ default_zodb_path }} tidstorage = {{ tidstorage_repozo_path }} [monitor-instance-parameter] monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }} monitor-httpd-port = {{ next_port() }} monitor-title = {{ slapparameter_dict['name'] }} password = {{ slapparameter_dict['monitor-passwd'] }} [buildout] extends = {{ logrotate_cfg }} {{ parameter_dict['template-monitor'] }} parts += {{ part_list | join('\n ') }} publish