Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Lukas Niegsch
slapos
Commits
f1c3a7ff
Commit
f1c3a7ff
authored
Aug 09, 2013
by
Romain Courteaud
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
Provide RSS feeds about the backup status.
parent
a07dffad
Changes
7
Hide whitespace changes
Inline
Side-by-side
Showing
7 changed files
with
266 additions
and
5 deletions
+266
-5
software/backupserver/instance-pullrdiffbackup.cfg.in
software/backupserver/instance-pullrdiffbackup.cfg.in
+51
-0
software/backupserver/software.cfg
software/backupserver/software.cfg
+42
-4
software/backupserver/status2rss.py
software/backupserver/status2rss.py
+46
-0
software/backupserver/template-backup-script.sh.in
software/backupserver/template-backup-script.sh.in
+17
-1
software/backupserver/template-nginx-service.sh.in
software/backupserver/template-nginx-service.sh.in
+18
-0
software/backupserver/template-nginx.cfg.in
software/backupserver/template-nginx.cfg.in
+75
-0
software/backupserver/template-update-rss.sh.in
software/backupserver/template-update-rss.sh.in
+17
-0
No files found.
software/backupserver/instance-pullrdiffbackup.cfg.in
View file @
f1c3a7ff
...
@@ -12,6 +12,7 @@ etc = $${buildout:directory}/etc
...
@@ -12,6 +12,7 @@ etc = $${buildout:directory}/etc
bin = $${buildout:directory}/bin
bin = $${buildout:directory}/bin
srv = $${buildout:directory}/srv
srv = $${buildout:directory}/srv
var = $${buildout:directory}/var
var = $${buildout:directory}/var
run = $${:var}/run
log = $${:var}/log
log = $${:var}/log
# scripts = $${:etc}/run
# scripts = $${:etc}/run
services = $${:etc}/service
services = $${:etc}/service
...
@@ -20,8 +21,11 @@ cron-lines = $${:etc}/cron.lines
...
@@ -20,8 +21,11 @@ cron-lines = $${:etc}/cron.lines
crontabs = $${:etc}/crontabs
crontabs = $${:etc}/crontabs
cronstamps = $${:etc}/cronstamps
cronstamps = $${:etc}/cronstamps
backup = $${:srv}/backup
backup = $${:srv}/backup
status = $${:srv}/status
backupscript = $${:etc}/backup
backupscript = $${:etc}/backup
www = $${:srv}/www
home = $${:etc}/home
home = $${:etc}/home
ssl = $${:etc}/ssl
ssh = $${:home}/.ssh
ssh = $${:home}/.ssh
#################################
#################################
...
@@ -91,6 +95,7 @@ storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
...
@@ -91,6 +95,7 @@ storage-path = {{ '$${' ~ slave_reference }}-backup-public_key:key}
recipe = slapos.cookbook:publish
recipe = slapos.cookbook:publish
-slave-reference = {{ slave_reference }}
-slave-reference = {{ slave_reference }}
authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:readline}
authorized_key = {{ '$${' ~ slave_reference }}-backup-read-public_key:readline}
rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/{{ '$${' ~ slave_reference }}-backup-script:status_name}.rss
{% do part_list.append("%s-backup-publish" % slave_reference) -%}
{% do part_list.append("%s-backup-publish" % slave_reference) -%}
[{{ slave_reference }}-backup-script]
[{{ slave_reference }}-backup-script]
...
@@ -105,6 +110,8 @@ hostname = {{ hostname }}
...
@@ -105,6 +110,8 @@ hostname = {{ hostname }}
include = {{ include_string }}
include = {{ include_string }}
exclude_string = {{ exclude_string }}
exclude_string = {{ exclude_string }}
remote_schema = {{ remote_schema }}
remote_schema = {{ remote_schema }}
status_name = {{ slave_reference }}_status.txt
status_log = $${directory:status}/$${:status_name}
[{{ slave_reference }}-backup-crontab-line]
[{{ slave_reference }}-backup-crontab-line]
recipe = slapos.recipe.template
recipe = slapos.recipe.template
...
@@ -121,6 +128,25 @@ frequency = {{ frequency }}
...
@@ -121,6 +128,25 @@ frequency = {{ frequency }}
#################################
#################################
# Generate crontab file
# Generate crontab file
#################################
#################################
[update-rss-script]
recipe = slapos.recipe.template
url = ${template-update-rss-script:output}
output = $${directory:etc}/$${:_buildout_section_name_}
mode = 0700
global_rss = $${slap-connection:computer-id}-$${slap-connection:partition-id}.rss
[update-rss-crontab-line]
recipe = slapos.recipe.template
url = ${template-crontab-line:output}
output = $${directory:cron-lines}/$${:_buildout_section_name_}
mode = 0600
script = $${update-rss-script:output}
frequency = */5 * * * *
{% do crontab_line_list.append("$${update-rss-crontab-line:output}") -%}
[publish-global-rss]
recipe = slapos.cookbook:publish
rss = https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${update-rss-script:global_rss}
{% set crontab_line_list_string = " ".join(crontab_line_list) -%}
{% set crontab_line_list_string = " ".join(crontab_line_list) -%}
[activate-crontab-file]
[activate-crontab-file]
...
@@ -129,12 +155,37 @@ recipe = plone.recipe.command
...
@@ -129,12 +155,37 @@ recipe = plone.recipe.command
stop-on-error = true
stop-on-error = true
command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
command = ${coreutils-output:cat} ${template-crontab:output} {{ crontab_line_list_string }} | ${dcron-output:crontab} -c $${directory:crontabs} -
#################################
# Nginx service
#################################
[nginx-service]
recipe = slapos.recipe.template
url = ${template-nginx-service:output}
output = $${directory:services}/nginx
mode = 0700
virtual-depends =
$${nginx-configuration:ip}
[nginx-configuration]
recipe = slapos.recipe.template
url = ${template-nginx-configuration:output}
output = $${directory:etc}/nginx.cfg
mode = 0600
access_log = $${directory:log}/nginx-access.log
error_log = $${directory:log}/nginx-error.log
ip = $${slap-network-information:global-ipv6}
port = 9443
ssl_key = $${directory:ssl}/nginx.key
ssl_csr = $${directory:ssl}/nginx.csr
ssl_crt = $${directory:ssl}/nginx.crt
# Add parts generated by template
# Add parts generated by template
[buildout]
[buildout]
parts =
parts =
dcron-service
dcron-service
nginx-service
activate-crontab-file
activate-crontab-file
publish-global-rss
{% for part in part_list -%}
{% for part in part_list -%}
{{ ' %s' % part }}
{{ ' %s' % part }}
{% endfor -%}
{% endfor -%}
...
...
software/backupserver/software.cfg
View file @
f1c3a7ff
...
@@ -3,7 +3,8 @@ extends =
...
@@ -3,7 +3,8 @@ extends =
../../component/dash/buildout.cfg
../../component/dash/buildout.cfg
../../component/dcron/buildout.cfg
../../component/dcron/buildout.cfg
../../component/logrotate/buildout.cfg
../../component/logrotate/buildout.cfg
# ../../component/nginx/buildout.cfg
../../component/openssl/buildout.cfg
../../component/nginx/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/rdiff-backup/buildout.cfg
# ../../component/duplicity/buildout.cfg
# ../../component/duplicity/buildout.cfg
# ../../component/git/buildout.cfg
# ../../component/git/buildout.cfg
...
@@ -15,11 +16,13 @@ extends =
...
@@ -15,11 +16,13 @@ extends =
../../stack/slapos.cfg
../../stack/slapos.cfg
parts =
parts =
extra-eggs
rdiff-backup
rdiff-backup
# duplicity
# duplicity
dcron
dcron
logrotate
logrotate
# nginx
nginx
openssl
# git
# git
# subversion
# subversion
rsync
rsync
...
@@ -30,6 +33,12 @@ parts =
...
@@ -30,6 +33,12 @@ parts =
template-crontab-line
template-crontab-line
slapos-cookbook
slapos-cookbook
[extra-eggs]
recipe = zc.recipe.egg
interpreter = pythonforrssgen
eggs =
PyRSS2Gen
[networkcache]
[networkcache]
# signature certificates of the following uploaders.
# signature certificates of the following uploaders.
# Romain Courteaud
# Romain Courteaud
...
@@ -163,6 +172,20 @@ signature-certificate-list =
...
@@ -163,6 +172,20 @@ signature-certificate-list =
##########################################################
##########################################################
# Service startup scripts and configuration files
# Service startup scripts and configuration files
##########################################################
##########################################################
[template-nginx-service]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx-service.sh.in
md5sum = 5c94d952305552dcbeaeaeb27dd28f3b
output = ${buildout:directory}/template-nginx-service.sh.in
mode = 0644
[template-nginx-configuration]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-nginx.cfg.in
md5sum = 1cf80a510a9ba9f6d9f12e2ce828615e
output = ${buildout:directory}/template-nginx.cfg.in
mode = 0644
[template-dcron-service]
[template-dcron-service]
recipe = slapos.recipe.template
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-dcron-service.sh.in
url = ${:_profile_base_location_}/template-dcron-service.sh.in
...
@@ -173,7 +196,7 @@ mode = 0644
...
@@ -173,7 +196,7 @@ mode = 0644
[template-backup-script]
[template-backup-script]
recipe = slapos.recipe.template
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-backup-script.sh.in
url = ${:_profile_base_location_}/template-backup-script.sh.in
md5sum =
19c3a72913cc285bf9745ab2f869018e
md5sum =
ce63dd27aaea8b64f446a7aef9406873
output = ${buildout:directory}/template-backup-script.sh.in
output = ${buildout:directory}/template-backup-script.sh.in
mode = 0644
mode = 0644
...
@@ -191,13 +214,27 @@ md5sum = 072be0fd04896880c931d44d8eabde37
...
@@ -191,13 +214,27 @@ md5sum = 072be0fd04896880c931d44d8eabde37
output = ${buildout:directory}/template-crontab.in
output = ${buildout:directory}/template-crontab.in
mode = 0644
mode = 0644
[status2rss]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/status2rss.py
md5sum = 138c96e0836f2b06414b98ba2643f21c
output = ${buildout:directory}/status2rss.py
mode = 0644
[template-update-rss-script]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/template-update-rss.sh.in
md5sum = 529058c54e873ab26f7920c868b23c50
output = ${buildout:directory}/template-update-rss.sh.in
mode = 0644
##########################################################
##########################################################
# Buildout instance.cfg templates
# Buildout instance.cfg templates
##########################################################
##########################################################
[template-pullrdiffbackup]
[template-pullrdiffbackup]
recipe = slapos.recipe.template
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
url = ${:_profile_base_location_}/instance-pullrdiffbackup.cfg.in
md5sum =
213256229360f57c778308825b161321
md5sum =
935ee60f0454ab50df74d0bcc3364d37
output = ${buildout:directory}/template-pullrdiffbackup.cfg
output = ${buildout:directory}/template-pullrdiffbackup.cfg
mode = 0644
mode = 0644
...
@@ -223,6 +260,7 @@ slapos.recipe.build = 0.11.6
...
@@ -223,6 +260,7 @@ slapos.recipe.build = 0.11.6
slapos.recipe.cmmi = 0.1.1
slapos.recipe.cmmi = 0.1.1
slapos.recipe.template = 2.4.2
slapos.recipe.template = 2.4.2
zc.recipe.egg = 1.3.2
zc.recipe.egg = 1.3.2
PyRSS2Gen = 1.1
# Required by:
# Required by:
# slapos.core==0.35.1
# slapos.core==0.35.1
...
...
software/backupserver/status2rss.py
0 → 100644
View file @
f1c3a7ff
import
datetime
import
uuid
import
PyRSS2Gen
import
sys
from
email.utils
import
parsedate_tz
,
mktime_tz
import
base64
# Based on http://thehelpfulhacker.net/2011/03/27/a-rss-feed-for-your-crontabs/
# ### Defaults
TITLE
=
sys
.
argv
[
1
]
LINK
=
sys
.
argv
[
2
]
DESCRIPTION
=
TITLE
items
=
[]
while
1
:
try
:
line
=
sys
.
stdin
.
readline
()
except
KeyboardInterrupt
:
break
if
not
line
:
break
time
,
desc
=
line
.
split
(
','
,
1
)
rss_item
=
PyRSS2Gen
.
RSSItem
(
title
=
desc
,
description
=
"%s, %s"
%
(
time
,
desc
),
link
=
LINK
,
pubDate
=
datetime
.
datetime
.
fromtimestamp
(
mktime_tz
(
parsedate_tz
(
time
))),
guid
=
PyRSS2Gen
.
Guid
(
base64
.
b64encode
(
"%s, %s"
%
(
time
,
desc
)))
)
items
.
append
(
rss_item
)
### Build the rss feed
rss_feed
=
PyRSS2Gen
.
RSS2
(
title
=
TITLE
,
link
=
LINK
,
description
=
DESCRIPTION
,
lastBuildDate
=
datetime
.
datetime
.
utcnow
(),
items
=
items
)
print
rss_feed
.
to_xml
()
software/backupserver/template-backup-script.sh.in
View file @
f1c3a7ff
...
@@ -4,7 +4,13 @@
...
@@ -4,7 +4,13 @@
export HOME=$${directory:home}
export HOME=$${directory:home}
set -e
# Clean status file (no history needed)
${coreutils-output:rm} -f $${:status_log}
# Inform about beginning of backup
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup running" >> $${:status_log}
# set -e
cd $${:datadirectory}
cd $${:datadirectory}
${rdiff-backup-output:rdiff-backup} \
${rdiff-backup-output:rdiff-backup} \
$${:exclude_string} \
$${:exclude_string} \
...
@@ -13,5 +19,15 @@ ${rdiff-backup-output:rdiff-backup} \
...
@@ -13,5 +19,15 @@ ${rdiff-backup-output:rdiff-backup} \
--remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
--remote-schema '${dropbear-output:ssh} -T -y -i $${:sshkey} %s $${:remote_schema}' \
$${:connection}::/ ./
$${:connection}::/ ./
RESULT=$?
# Inform about backup status
if [ $RESULT == 0 ]
then
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup success" >> $${:status_log}
else
${coreutils-output:echo} "`${coreutils-output:date} -u`, $${:hostname} backup failed" >> $${:status_log}
fi
# python scripts/verify_with_sudo.py ./ $${:connection}:/
# python scripts/verify_with_sudo.py ./ $${:connection}:/
software/backupserver/template-nginx-service.sh.in
0 → 100644
View file @
f1c3a7ff
#!${dash-output:dash}
# BEWARE: This file is operated by slapgrid
# BEWARE: It will be overwritten automatically
if [ ! -e $${nginx-configuration:ssl_crt} ]
then
${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl_key} 2048
${openssl-output:openssl} req -new \
-subj "/C=AA/ST=Denial/L=Nowhere/O=Dis/CN=$${nginx-configuration:ip}" \
-key $${nginx-configuration:ssl_key} -out $${nginx-configuration:ssl_csr}
${openssl-output:openssl} x509 -req -days 365 \
-in $${nginx-configuration:ssl_csr} \
-signkey $${nginx-configuration:ssl_key} \
-out $${nginx-configuration:ssl_crt}
fi
exec ${nginx-output:nginx} \
-c $${nginx-configuration:output}
software/backupserver/template-nginx.cfg.in
0 → 100644
View file @
f1c3a7ff
daemon off; # run in the foreground so supervisord can look after it
worker_processes 4;
pid $${directory:run}/nginx.pid;
events {
worker_connections 768;
# multi_accept on;
}
http {
##
# Basic Settings
##
sendfile on;
tcp_nopush on;
tcp_nodelay on;
keepalive_timeout 65;
types_hash_max_size 2048;
server_tokens off;
# server_names_hash_bucket_size 64;
# server_name_in_redirect off;
default_type application/octet-stream;
include ${nginx-output:mime};
##
# Logging Settings
##
access_log $${nginx-configuration:access_log};
error_log $${nginx-configuration:error_log};
##
# Gzip Settings
##
gzip on;
gzip_disable "msie6";
# gzip_vary on;
# gzip_proxied any;
# gzip_comp_level 6;
# gzip_buffers 16 8k;
# gzip_http_version 1.1;
# gzip_types text/plain text/css application/json application/x-javascript text/xml application/xml application/xml+rss text/javascript;
server {
listen [$${nginx-configuration:ip}]:$${nginx-configuration:port};
ssl on;
ssl_certificate $${nginx-configuration:ssl_crt};
ssl_certificate_key $${nginx-configuration:ssl_key};
## Only allow GET and HEAD request methods
if ($request_method !~ ^(GET|HEAD)$ ) {
return 444;
}
## Serve an error 204 (No Content) for favicon.ico
location = /favicon.ico {
return 204;
}
location /
{
root $${directory:www};
# index index.html;
}
}
}
software/backupserver/template-update-rss.sh.in
0 → 100644
View file @
f1c3a7ff
#!${dash-output:dash}
STATUS_DIR=$${directory:status}
RSS_DIR=$${directory:www}
CAT=${coreutils-output:cat}
PYTHON=${buildout:directory}/bin/${extra-eggs:interpreter}
STATUS2RSS=${status2rss:output}
BASENAME=${coreutils-output:basename}
for status in $STATUS_DIR/*
do
NAME=`$BASENAME $status`
$CAT $status | $PYTHON $STATUS2RSS "Backup status $NAME" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$NAME.rss" > $RSS_DIR/$NAME.rss
done
$CAT $STATUS_DIR/* | $PYTHON $STATUS2RSS "Full backup status $${:global_rss}" "https://[$${nginx-configuration:ip}]:$${nginx-configuration:port}/$${:global_rss}" > $RSS_DIR/$${:global_rss}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment