instance-pull-backup.cfg.in 11.3 KB
Newer Older
1 2 3
[buildout]

parts =
4
  publish-connection-information
5 6
  pbs
  logrotate
7
  logrotate-entry-notifier
8 9
  cron
  cron-entry-logrotate
10
  pbs-sshkeys-authority
11
  sshkeys-openssh
12
  backup-checksum-integrity-promise
13
  resilient-genstatrss-wrapper
14
  pbs-push-history-log
15
  backup-signature-link
16
  cron-pbs-status-feed
17
  pull-push-stalled-promise
18
  notifier-feed-status-promise
19

20
## Monitor for pbs
21
  monitor-base
22
  monitor-check-resilient-feed-file
23

24
extends = ${monitor2-template:rendered}
25 26 27 28
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true

29

30 31 32 33 34 35 36 37 38 39 40 41 42 43 44
#----------------
#--
#-- Creation of all needed directories.

[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc
srv = $${buildout:directory}/srv
bin = $${buildout:directory}/bin
tmp = $${buildout:directory}/tmp
var = $${buildout:directory}/var

[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log
45
services = $${rootdirectory:etc}/service
46 47 48
run = $${rootdirectory:var}/run
backup = $${rootdirectory:srv}/backup
promises = $${rootdirectory:etc}/promise
49
ssh-home = $${rootdirectory:etc}/ssh
50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65
notifier = $${rootdirectory:etc}/notifier

[directory]
recipe = slapos.cookbook:mkdirectory
logrotate-entries = $${rootdirectory:etc}/logrotate.d
logrotate-backup = $${basedirectory:backup}/logrotate
cronstamps = $${rootdirectory:etc}/cronstamps
cron-entries = $${rootdirectory:etc}/cron.d
crontabs = $${rootdirectory:etc}/crontabs
cronoutput = $${basedirectory:log}/cron-ouput
pbs-backup = $${basedirectory:backup}/pbs
sshkeys = $${rootdirectory:srv}/sshkeys
pbs-wrappers = $${rootdirectory:bin}/pbs
dot-ssh = $${basedirectory:ssh-home}/.ssh
notifier-feeds = $${basedirectory:notifier}/feeds
notifier-callbacks = $${basedirectory:notifier}/callbacks
66
notifier-status-items = $${basedirectory:notifier}/status-items
67
monitor-resilient = $${monitor-directory:private}/resilient
68 69 70 71 72


#----------------
#--
#-- Set up the equeue and notifier.
73

74 75 76
[equeue]
recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock
77
lockfile = $${basedirectory:run}/equeue.lock
78 79 80 81 82
log = $${basedirectory:log}/equeue.log
database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue

83 84 85 86 87 88
[notifier-port]
recipe = slapos.cookbook:free_port
minimum = 8088
maximum = 8097
ip = $${notifier:host}

89 90
# notifier.notify adds the [exporter, notifier] to the execution queue
# notifier.notify.callback sets up a callback
91
[notifier]
92 93 94 95 96 97
recipe = slapos.recipe.template:jinja2
template = ${template-wrapper:output}
rendered = $${:wrapper}
wrapper = $${basedirectory:services}/notifier
mode = 0700
command = ${buildout:bin-directory}/pubsubserver --callbacks $${directory:notifier-callbacks} --feeds $${directory:notifier-feeds} --equeue-socket $${equeue:socket} --logfile $${basedirectory:log}/notifier.log $${:host} $${:port}
98
host = $${slap-network-information:global-ipv6}
99
port = $${notifier-port:port}
100 101 102
context =
  key content notifier:command

103 104 105 106 107 108 109
[logrotate-entry-equeue]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = equeue
log = $${equeue:log}
frequency = daily
rotate-num = 30
110

111

112 113 114 115
#----------------
#--
#-- The pull-backup-server contains every backup (incremental)
#-- to prevent a corrupt dump from destroying everything.
116

117 118 119
[pbs]
<= notifier
recipe = slapos.cookbook:pbs
120
client = true
121 122 123 124
feeds = $${directory:notifier-feeds}
callbacks = $${directory:notifier-callbacks}
equeue-socket = $${equeue:socket}
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
125
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
126
sshclient-binary = $${openssh-client:wrapper-path}
127 128
known-hosts = $${directory:dot-ssh}/known_hosts
promises-directory = $${basedirectory:promises}
129
directory = $${directory:pbs-backup}
130 131
cron-entries = $${cron:cron-entries}
wrappers-directory = $${directory:pbs-wrappers}
132
run-directory = $${basedirectory:run}
133 134
# XXX: this should be named "notifier-host"
notifier-url = http://[$${notifier:host}]:$${notifier:port}
135
slave-instance-list = $${slap-parameter:slave_instance_list}
136
ignore-known-hosts-file = $${slap-parameter:ignore-known-hosts-file}
137 138 139 140
# To get a verbose feed about PBS state
instance-root-name = $${instance-info-parameters:root-name}
log-url = $${publish:monitor-base-url}/private/notifier/
status-item-directory = $${directory:notifier-status-items}
141

142 143
[pbs-resilient-status-feed]
recipe = slapos.cookbook:wrapper
144 145
command-line = ${buildout:directory}/bin/generatefeed --output $${:feed-path} --status-item-path $${pbs:status-item-directory} --title "Status feed for $${instance-info-parameters:root-name}-PBS" --link $${pbs:log-url}
feed-path = $${directory:monitor-resilient}/pbs-status-rss
146 147 148 149 150 151
wrapper-path = $${rootdirectory:bin}/resilient-genstatusrss.py

[cron-pbs-status-feed]
<= cron
recipe = slapos.cookbook:cron.d
name = resilient-pbs-status-feed
152
frequency = */5 * * * *
153
command = $${pbs-resilient-status-feed:wrapper-path}
154

155 156 157
#----------------
#--
#-- Deploy cron.
158 159 160 161 162 163 164 165 166 167

[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond

168 169 170 171 172 173 174 175 176 177
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log


#----------------
#--
#-- Deploy logrotate.

178 179 180 181 182 183 184 185 186 187
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}

[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
188
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
189 190 191 192 193 194 195 196 197
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status

198
[logrotate-entry-cron]
199 200 201 202 203 204 205
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = cron
log = $${cron-simplelogger:log}
frequency = daily
rotate-num = 30

206 207 208 209 210 211 212 213 214 215 216 217
[logrotate-entry-notifier]
recipe = collective.recipe.template
mode = 600
input = inline:
  $${directory:notifier-feeds}/* {
    rotate 5
    weekly
    nocompress
    missingok
    olddir $${directory:logrotate-backup}
  }
output = $${logrotate:logrotate-entries}/notifier
218

219 220 221
#----------------
#--
#-- sshkeys
222

223
[sshkeys-directory]
224
recipe = slapos.cookbook:mkdirectory
225 226
requests = $${directory:sshkeys}/openssl-requests
keys = $${directory:sshkeys}/openssl-keys
227

228
[pbs-sshkeys-authority]
229 230 231
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
232
wrapper = $${basedirectory:services}/pbs_sshkeys_authority
233
keygen-binary = ${openssh:location}/bin/ssh-keygen
234

235
[sshkeys-openssh]
236
<= pbs-sshkeys-authority
237
recipe = slapos.cookbook:sshkeys_authority.request
238
name = pbs-client
239
type = rsa
240 241 242
executable = $${openssh-client:wrapper-path}
public-key = $${openssh-client:identity-file}.pub
private-key = $${openssh-client:identity-file}
243
wrapper = $${rootdirectory:bin}/do_backup
244 245


246 247
#----------------
#--
248
#-- OpenSSH.
249

250 251
[openssh-client]
recipe = slapos.cookbook:wrapper
252
home = $${basedirectory:ssh-home}
253 254 255 256
identity-file = $${:home}/id_rsa
command-line = ${openssh:location}/bin/ssh -T -o "UserKnownHostsFile $${pbs:known-hosts}" -i $${:identity-file}
wrapper-path = $${rootdirectory:bin}/ssh
parameters-extra = true
257 258 259 260 261


#----------------
#--
#-- Slave instance list (empty default).
262 263 264 265 266
[htpasswd]
recipe = slapos.cookbook:generate.password
storage-path = $${directory:etc}/.monitor_user
bytes = 8
username = admin
267 268 269

[slap-parameter]
slave_instance_list = []
270
ignore-known-hosts-file = false
271 272 273 274 275
monitor-cors-domains = 
monitor-httpd-port = 8070
monitor-title = PBS Instance
monitor-password = $${htpasswd:passwd}
monitor-username = $${htpasswd:username}
276

277 278 279 280
#----------------
#--
#-- Resiliency promises.

281
[backup-checksum-integrity-promise]
282 283
recipe = slapos.recipe.template:jinja2
template = inline:
284 285 286 287 288 289 290 291 292
  #!${dash:location}/bin/dash
  # Raise an error if signatures are different
  # Error cannot be deduced if files do not exist
  cd $${directory:pbs-backup}
  if [ ! -f "proof.signature" ]; then exit 0; fi
  backup_signature=$(find . -maxdepth 2 -name backup.signature)
  if [ -z "$backup_signature" ]; then
    exit 0;
  else
293 294 295 296 297 298 299
    diff -q "proof.signature" "$backup_signature";
    if [ "$?" -eq 0 ]; then
      exit 0;
    else
      echo "Signature file is not the same before and after transfer"
      exit 1
    fi
300
  fi
301
rendered = $${basedirectory:promises}/backup-checksum-integrity
302 303
mode = 700

304 305 306 307 308
[resilient-genstatrss-wrapper]
recipe = slapos.cookbook:wrapper
# XXX - hard-coded Urls
command-line = ${buildout:directory}/bin/rdiffbackup.genstatrss --output '$${monitor-directory:public}/resilient-feed' --rdiff_backup_data_folder '$${pbs:rdiff-backup-data-folder}' --feed_url '$${monitor-conf-parameters:base-url}/public/resilient-feed'
wrapper-path = $${directory:bin}/resilient-genstatrss.py
309

310 311 312 313
[pbs-push-history-log]
recipe = cns.recipe.symlink
symlink = $${pbs:rdiff-backup-data-folder}/restore.log = $${basedirectory:log}/pbs-push-history-log

314 315
[backup-signature-link]
recipe = cns.recipe.symlink
316
symlink = $${directory:pbs-backup}/proof.signature = $${directory:monitor-resilient}/backup.signature
317

318 319
[pull-push-stalled-promise]
recipe = slapos.cookbook:wrapper
320 321
# # time-buffer is 24h (+1h of latitude)
command-line = ${buildout:bin-directory}/check-feed-as-promise --feed-path $${pbs-resilient-status-feed:feed-path} --title --ok-pattern 'OK' --time-buffer 90000
322 323
wrapper-path = $${basedirectory:promises}/stalled-pull-push

324 325 326 327 328 329 330 331 332 333
[notifier-feed-status-promise]
recipe = slapos.recipe.template:jinja2
template = ${notifier-feed-promise-template:target}
rendered = $${basedirectory:promises}/notifier-feed-check-malformed-or-failure.py
mode = 700
context =
  key notifier_feed_directory directory:notifier-feeds
  raw base_url http://[$${notifier:host}]:$${notifier:port}/get/
  raw python_executable ${buildout:executable}

334 335 336 337
#----------------
#--
#-- Publish instance parameters.

338
[publish-connection-information]
339
recipe = slapos.cookbook:publish
340
ssh-key = $${sshkeys-openssh:public-key-value}
341 342
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
343 344 345 346
monitor-base-url = $${publish:monitor-base-url}
monitor-url = $${publish:monitor-url}
monitor-user = $${publish:monitor-user}
monitor-password = $${publish:monitor-password}
347 348 349 350 351

#----------------
#--
#-- Monitor

352 353 354 355 356 357
[monitor-instance-parameter]
monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
monitor-title = $${slap-parameter:monitor-title}
cors-domains = $${slap-parameter:monitor-cors-domains}
username = $${slap-parameter:monitor-username}
password = $${slap-parameter:monitor-password}
358

359 360 361 362
[monitor-conf-parameters]
private-path-list += 
  $${directory:logrotate-backup}
  $${basedirectory:log}
363 364 365 366

[monitor-check-resilient-feed-file]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
367
rendered = $${monitor-directory:reports}/check-create-resilient-feed-files
368 369 370
mode = 700
context =
  key input_feed_directory directory:notifier-feeds
371
  key monitor_feed_directory monitor-directory:public
372
  key base_url publish-connection-information:feeds-url
373
  raw python_executable ${buildout:executable}