Commit 50d5e186 authored by Julien Muchembled's avatar Julien Muchembled

slaprunner: partial rewrite of import/export scrips in Python

Further rewrite should be done but at least there's no awk code anymore.
parent f92aefb0
No related merge requests found
[buildout]
parts = gawk
[gawk]
recipe = slapos.recipe.cmmi
url = http://ftp.gnu.org/gnu/gawk/gawk-4.1.4.tar.gz
md5sum = f20c94ca51b6ebfc9bffb90f95c8ffbb
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
# not need these here). # not need these here).
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = e89d25c1bb7bf507bd96973d444934a6 md5sum = 78e4c796b15a9c5b9b41c79a3c995ee7
[template-runner] [template-runner]
filename = instance-runner.cfg filename = instance-runner.cfg
...@@ -23,19 +23,19 @@ md5sum = c76c66c331f1f0d9f416c56ead53e804 ...@@ -23,19 +23,19 @@ md5sum = c76c66c331f1f0d9f416c56ead53e804
[template-runner-import-script] [template-runner-import-script]
filename = template/runner-import.sh.jinja2 filename = template/runner-import.sh.jinja2
md5sum = feee1dc29d160e0d796dab0ba767745f md5sum = 40e839e76d03fabefb1859ea72383529
[instance-runner-import] [instance-runner-import]
filename = instance-runner-import.cfg.in filename = instance-runner-import.cfg.in
md5sum = 70931f214b3f7c3992e29c150a30d3cf md5sum = 5d49f4fe6de7bd02db0067d0aeac6de4
[template-runner-export-script] [template-runner-export-script]
filename = template/runner-export.sh.jinja2 filename = template/runner-export.sh.jinja2
md5sum = b921435ab5a8c408026e74d9a40eea44 md5sum = 02023f89b622c49aee42130f839846a7
[instance-runner-export] [instance-runner-export]
filename = instance-runner-export.cfg.in filename = instance-runner-export.cfg.in
md5sum = 546b368baaed84bd2d3be7d6b68a5030 md5sum = 22a5dfe9bfc8533f8eab851e0996a6d7
[template-resilient] [template-resilient]
filename = instance-resilient.cfg.jinja2 filename = instance-resilient.cfg.jinja2
......
...@@ -2,12 +2,10 @@ ...@@ -2,12 +2,10 @@
extends = extends =
buildout.hash.cfg buildout.hash.cfg
../../component/bash/buildout.cfg ../../component/bash/buildout.cfg
../../component/coreutils/buildout.cfg
../../component/busybox/buildout.cfg ../../component/busybox/buildout.cfg
../../component/curl/buildout.cfg ../../component/curl/buildout.cfg
../../component/dash/buildout.cfg ../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg ../../component/dcron/buildout.cfg
../../component/gawk/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/tig/buildout.cfg ../../component/tig/buildout.cfg
../../component/logrotate/buildout.cfg ../../component/logrotate/buildout.cfg
......
...@@ -62,11 +62,6 @@ ip = ${slaprunner:ipv4} ...@@ -62,11 +62,6 @@ ip = ${slaprunner:ipv4}
[supervisord] [supervisord]
port = ${supervisord-free-port:port} port = ${supervisord-free-port:port}
[exporter-configuration]
coreutils-location = {{ dumps(parameter_dict['coreutils-location']) }}
gawk-location = {{ dumps(parameter_dict['gawk-location']) }}
backup_wait_time = {{ dumps(parameter_dict['backup_wait_time']) }}
[exporter] [exporter]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = {{ exporter_script_path }} template = {{ exporter_script_path }}
...@@ -75,8 +70,10 @@ rendered = ${directory:bin}/${slap-parameter:namebase}-exporter ...@@ -75,8 +70,10 @@ rendered = ${directory:bin}/${slap-parameter:namebase}-exporter
wrapper = ${:rendered} wrapper = ${:rendered}
mode = 700 mode = 700
context = context =
import sys sys
import easy_install zc.buildout.easy_install
section directory directory section directory directory
section parameter_dict exporter-configuration raw backup_wait_time {{ backup_wait_time }}
raw output_log_file ${directory:log}/resilient.log raw output_log_file ${directory:log}/resilient.log
raw shell_binary {{ bash_executable_location }} raw shell_binary {{ bash_executable_location }}
raw rsync_binary {{ rsync_executable_location }} raw rsync_binary {{ rsync_executable_location }}
......
...@@ -72,10 +72,6 @@ maximum = 22241 ...@@ -72,10 +72,6 @@ maximum = 22241
[cron-entry-prepare-software] [cron-entry-prepare-software]
recipe = recipe =
[importer-configuration]
coreutils-location = ${coreutils:location}
gawk-location = ${gawk:location}
[importer] [importer]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${template-runner-import-script:location}/${template-runner-import-script:filename} template = ${template-runner-import-script:location}/${template-runner-import-script:filename}
...@@ -89,12 +85,13 @@ restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-ba ...@@ -89,12 +85,13 @@ restore-error-message-file = $${directory:srv}/$${:restore-error-message-file-ba
restore-error-message-file-basename = importer-error-message-file restore-error-message-file-basename = importer-error-message-file
resilient-log-basename = resilient.log resilient-log-basename = resilient.log
context = context =
import sys sys
import easy_install zc.buildout.easy_install
key backend_url slaprunner:access-url key backend_url slaprunner:access-url
key ipv4 slaprunner:ipv4 key ipv4 slaprunner:ipv4
key ipv6 slaprunner:ipv6 key ipv6 slaprunner:ipv6
key proxy_port slaprunner:proxy_port key proxy_port slaprunner:proxy_port
key instance_folder slaprunner:instance_root key instance_folder slaprunner:instance_root
section parameter_dict importer-configuration
section directory directory section directory directory
section supervisord supervisord section supervisord supervisord
raw output_log_file $${directory:log}/$${:resilient-log-basename} raw output_log_file $${directory:log}/$${:resilient-log-basename}
......
...@@ -47,11 +47,6 @@ import-list = file parts :template-parts-destination ...@@ -47,11 +47,6 @@ import-list = file parts :template-parts-destination
file replicated :template-replicated-destination file replicated :template-replicated-destination
mode = 0644 mode = 0644
[template-runner-export-configuration]
coreutils-location = ${coreutils:location}
gawk-location = ${gawk:location}
backup_wait_time = ${exporter-default-configuration:backup_wait_time}
[template-runner-export] [template-runner-export]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = ${instance-runner-export:target} template = ${instance-runner-export:target}
...@@ -60,7 +55,7 @@ mode = 640 ...@@ -60,7 +55,7 @@ mode = 640
context = context =
key pbsready_export_template_path template-pbsready-export:rendered key pbsready_export_template_path template-pbsready-export:rendered
key template_runner_path instance-base-runner:rendered key template_runner_path instance-base-runner:rendered
section parameter_dict template-runner-export-configuration raw backup_wait_time ${exporter-default-configuration:backup_wait_time}
raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename} raw exporter_script_path ${template-runner-export-script:location}/${template-runner-export-script:filename}
raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename} raw monitor_check_resilient_feed_template_path ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
raw buildout_executable_location ${buildout:executable} raw buildout_executable_location ${buildout:executable}
......
...@@ -24,15 +24,8 @@ rsync () { ...@@ -24,15 +24,8 @@ rsync () {
set +x set +x
} }
relativise () {
while IFS= read -r line; do
if [ ! -z "$line" ]; then
'{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
fi
done
}
( (
# XXX: code duplication with runner-import.sh.jinja2
path=$srv_directory/runner path=$srv_directory/runner
backup_path=$backup_directory/runner/ backup_path=$backup_directory/runner/
cd "$path" cd "$path"
...@@ -41,31 +34,46 @@ relativise () { ...@@ -41,31 +34,46 @@ relativise () {
# Concatenate the exclude file of each partition of webrunner # Concatenate the exclude file of each partition of webrunner
# to create a global exclude file. # to create a global exclude file.
# Also, ignore all buildout-managed files. # Also, ignore all buildout-managed files.
( exclude=$({{ sys.executable }} - "$path" <<EOF
echo "*.sock" if 1:
echo "*.socket" import glob, errno, os, sys
echo "*.pid" sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
echo ".installed*.cfg" from zc.buildout.configparser import parse
for partition in "$path"/instance/slappart*; do path = sys.argv[1]
# So "relativise" can handle relative paths (which are expected to be relative to partition).
cd "$partition" def print_relative(path_list):
exclude_file=srv/exporter.exclude for p in path_list:
if [ -r "$exclude_file" ]; then p = p.strip()
relativise "$path" < "$exclude_file" if p:
fi print(os.path.relpath(p, path))
for installed in .installed*.cfg; do print("*.sock")
if [ -r "$installed" ]; then print("*.socket")
# Print every line from each __buildout_installed__ found. print("*.pid")
'{{ parameter_dict["gawk-location"] }}/bin/gawk' ' print(".installed*.cfg")
BEGIN { do_print = 0 } for partition in glob.glob(path + "/instance/slappart*"):
match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next } os.chdir(partition)
/^\S/ { do_print = 0; next } try:
match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] } with open("srv/exporter.exclude") as f:
' "$installed" | relativise "$path" exclude = f.readlines()
fi except IOError as e:
done if e.errno != errno.ENOENT:
done raise
) | rsync --exclude-from=- instance "$backup_path" else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$backup_path"
fi fi
test -d project && rsync project "$backup_path" test -d project && rsync project "$backup_path"
...@@ -116,6 +124,6 @@ if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then ...@@ -116,6 +124,6 @@ if diff "$tmp_backup_sum" "$tmp_filtered_signature"; then
exit 0 exit 0
fi fi
echo "ERROR: Some backups are not consistent, exporter should be re-run." echo "ERROR: Some backups are not consistent, exporter should be re-run."
echo "Let's sleep {{ parameter_dict['backup_wait_time'] }} minutes, to let the backup end..." echo "Let's sleep {{ backup_wait_time }} minutes, to let the backup end..."
sleep {{ parameter_dict["backup_wait_time"] }}m sleep {{ backup_wait_time }}m
exit 1 exit 1
...@@ -41,16 +41,9 @@ rsync () { ...@@ -41,16 +41,9 @@ rsync () {
set +x set +x
} }
relativise () {
while IFS= read -r line; do
if [ ! -z "$line" ]; then
'{{ parameter_dict["coreutils-location"] }}/bin/realpath' --quiet --canonicalize-missing --no-symlinks --relative-to="$1" "$line"
fi
done
}
log_message "Restoring WebRunner content..." log_message "Restoring WebRunner content..."
( (
# XXX: code duplication with runner-export.sh.jinja2
path=$srv_directory/runner path=$srv_directory/runner
backup_path=$backup_directory/runner/ backup_path=$backup_directory/runner/
cd "$backup_path" cd "$backup_path"
...@@ -59,31 +52,46 @@ log_message "Restoring WebRunner content..." ...@@ -59,31 +52,46 @@ log_message "Restoring WebRunner content..."
# Concatenate the exclude file of each partition of webrunner # Concatenate the exclude file of each partition of webrunner
# to create a global exclude file. # to create a global exclude file.
# Also, ignore all buildout-managed files. # Also, ignore all buildout-managed files.
( exclude=$({{ sys.executable }} - "$path" <<EOF
echo "*.sock" if 1:
echo "*.socket" import glob, errno, os, sys
echo "*.pid" sys.path[:0] = {{ repr(easy_install.buildout_and_setuptools_path) }}
echo ".installed*.cfg" from zc.buildout.configparser import parse
for partition in "$path"/instance/slappart*; do path = sys.argv[1]
# So "relativise" can handle relative paths (which are expected to be relative to partition).
cd "$partition" def print_relative(path_list):
exclude_file=srv/exporter.exclude for p in path_list:
if [ -r "$exclude_file" ]; then p = p.strip()
relativise "$path" < "$exclude_file" if p:
fi print(os.path.relpath(p, path))
for installed in .installed*.cfg; do print("*.sock")
if [ -r "$installed" ]; then print("*.socket")
# Print every line from each __buildout_installed__ found. print("*.pid")
'{{ parameter_dict["gawk-location"] }}/bin/gawk' ' print(".installed*.cfg")
BEGIN { do_print = 0 } for partition in glob.glob(path + "/instance/slappart*"):
match($0, /^__buildout_installed__\s*=\s*(\S.*)/, ary) { do_print = 1; print ary[1]; next } os.chdir(partition)
/^\S/ { do_print = 0; next } try:
match($0, /^\s+(\S.*)/, ary) { if (do_print) print ary[1] } with open("srv/exporter.exclude") as f:
' "$installed" | relativise "$path" exclude = f.readlines()
fi except IOError as e:
done if e.errno != errno.ENOENT:
done raise
) | rsync --exclude-from=- instance "$path" else:
print_relative(exclude)
for installed in glob.glob(".installed*.cfg"):
try:
with open(installed) as f:
installed = parse(f, installed)
except IOError as e:
if e.errno != errno.ENOENT:
raise
else:
for section in installed.itervalues():
print_relative(section.get(
'__buildout_installed__', '').splitlines())
EOF
)
echo "$exclude" |rsync --exclude-from=- instance "$path"
fi fi
test -d project && rsync project "$path" test -d project && rsync project "$path"
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment