Skip to content

slapos
slapos
Commit 4b05a7da authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

Merge branch 'master' into grid-computing

parents fb52bb07 2d7f2f73
Hide whitespace changes
Inline Side-by-side
Showing with 1633 additions and 561 deletions
CHANGES.txt
View file @ 4b05a7da
Changes
=======
0.68.1 (2012-10-03)
-------------------
* slaprunner: fix "logfile" parameter to "log_file"
0.68 (2012-10-02)
-----------------
* request.py: Remove useless calls to master, fix "update" method. [Cedric
de Saint Martin]
* Add webrunner test recipe. [Alain Takoudjou]
* Add logfile for slaprunner. [Cedric de Saint Martin]
* Fix check_url_available promise (syntax + checks + IPv6 support). [Cedric
de Saint Martin]
0.67 (2012-09-26)
-----------------
* Add check_page_content promise generator. [Cedric Le Ninivin]
* Fix check_url_available recipe. [Cedric de Saint Martin]
* Set up timezone database in mariab's mysql table so that we can use
timezone conversion function. [Kazuhiko Shiozaki]
* Add many resiliency-based recipes [Timothée Lacroix]
* Fix and unify request and requestoptional recipes [Cedric de Saint Martin]
* Fix Dropbear. [Antoine Catton]
0.66 (2012-09-10)
-----------------
......
component/gcc/buildout.cfg
View file @ 4b05a7da
......@@ -170,4 +170,4 @@ make-targets = install -j1
# until gcc will be simplified by using more robust build recipe (like
# slapos.recipe.build) each time any of parts which reuses this one gets updated
# the hack-revision have to be increased
hack-revision = 1
hack-revision = 2
component/ghostscript/buildout.cfg
View file @ 4b05a7da
......@@ -29,5 +29,5 @@ environment =
[ghostscript-9]
<= ghostscript-common
url = http://downloads.ghostscript.com/public/ghostscript-9.05.tar.bz2
md5sum = 8bcef1f33ddf8a4d12b2cf8da385c191
url = http://downloads.ghostscript.com/public/ghostscript-9.06.tar.bz2
md5sum = 46f9ebe40dc52755287b30704270db11
component/groonga/buildout.cfg
View file @ 4b05a7da
......@@ -7,12 +7,12 @@ parts =
[groonga]
recipe = hexagonit.recipe.cmmi
url = http://packages.groonga.org/source/groonga/groonga-2.0.5.tar.gz
md5sum = 6ea6634ceee97f701add89ee39f4c1da
url = http://packages.groonga.org/source/groonga/groonga-2.0.6.tar.gz
md5sum = 64467d1ab8d5e0df8348a636972a95a7
configure-options =
--disable-static
--disable-glibtest
--disable-benchmark
--disable-document
--disable-nginx
--disable-groonga-httpd
--without-mecab
component/haproxy/buildout.cfg
View file @ 4b05a7da
......@@ -9,8 +9,8 @@ parts = haproxy
[haproxy]
recipe = hexagonit.recipe.cmmi
url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.21.tar.gz
md5sum = f36d5d13fd4a44ab454eee5d06cb38a2
url = http://haproxy.1wt.eu/download/1.4/src/haproxy-1.4.22.tar.gz
md5sum = a0b007c76f6a78524f3b3dd5e704979c
configure-command = true
# If the system is running on Linux 2.6, we use "linux26" as the TARGET,
# otherwise use "generic".
......
component/java/buildout.cfg
View file @ 4b05a7da
......@@ -29,9 +29,9 @@ slapos_promisee =
directory:javaws
file:lib/rt.jar
file:bin/java
# http://java.com/en/download/manual.jsp?locale=en
x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59621 7466b0b86bef21e3e31ae578b2b9f472
x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=59623 c5cfb8393d257b51e7a70e56b7784ac9
# http://java.com/en/download/manual_v6.jsp
x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68284 7eda40e790de1a907d591b62949b6e72
x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68286 839548714931443ba89719a995ece846
script =
import glob
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
......@@ -47,48 +47,20 @@ script =
workdir = guessworkdir(extract_dir)
self.copyTree(glob.glob(os.path.join(workdir, "jre1.6.0_*"))[0], "%(location)s")
[java-sdk-1.6.0]
[java-re-1.7.0]
recipe = slapos.recipe.build
slapos_promisee =
directory:bin
directory:lib
directory:man
directory:plugin
directory:javaws
file:jre/lib/rt.jar
file:bin/java
x86 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-i586.bin bdb5f05bd20c6aa9a4729726191bf6fd
x86-64 = http://download.oracle.com/otn-pub/java/jdk/6u27-b07/jdk-6u27-linux-x64.bin 94f93a3ff03f824a238ecd79ad90433e
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
download_file = self.download(self.options['url'], self.options.get('md5sum'))
extract_dir = tempfile.mkdtemp(self.name)
os.chdir(extract_dir)
(download_dir, filename) = os.path.split(download_file)
auto_extract_bin = os.path.join(extract_dir, filename)
shutil.move(download_file, auto_extract_bin)
os.chmod(auto_extract_bin, 0777)
subprocess.call(["patch", auto_extract_bin, "-i", "${jdk-6u27-no-user-interaction-patch:location}/${jdk-6u27-no-user-interaction-patch:filename}"])
subprocess.call([auto_extract_bin])
self.cleanup_dir_list.append(extract_dir)
workdir = guessworkdir(extract_dir)
self.copyTree(os.path.join(workdir, "jdk1.6.0_27"), "%(location)s")
[java-sdk-1.7.0]
recipe = slapos.recipe.build
slapos_promisee =
directory:bin
directory:lib
directory:man
directory:jre
file:jre/lib/rt.jar
file:lib/rt.jar
file:bin/java
file:bin/javac
x86 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-i586.tar.gz f97244a104f03731e5ff69f0dd5a9927
x86-64 = http://download.oracle.com/otn-pub/java/jdk/7/jdk-7-linux-x64.tar.gz b3c1ef5faea7b180469c129a49762b64
# http://java.com/en/download/manual.jsp?locale=en
x86 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68234 ea99bedd9db33e9e2970f4b70abd1e4b
x86-64 = http://javadl.sun.com/webapps/download/AutoDL?BundleId=68236 5aa9bd26cdf1fa6afd2b15826b4ba139
script =
if not self.options.get('url'): self.options['url'], self.options['md5sum'] = self.options[guessPlatform()].split(' ')
extract_dir = self.extract(self.download(self.options['url'], self.options.get('md5sum')))
workdir = guessworkdir(extract_dir)
self.copyTree(workdir, "%(location)s")
component/mariadb/buildout.cfg
View file @ 4b05a7da
......@@ -59,9 +59,11 @@ environment =
LDFLAGS=-L${libaio:location}/lib
[mroonga-mariadb]
# mroonga - a storage engine for MySQL. It provides fast fulltext search feature to all MySQL users.
# http://mroonga.github.com/
recipe = hexagonit.recipe.cmmi
url = https://github.com/downloads/mroonga/mroonga/mroonga-2.05.tar.gz
md5sum = 6aee0e6ff948bba7c02854326afc11df
url = https://github.com/downloads/mroonga/mroonga/mroonga-2.06.tar.gz
md5sum = 89ce640a57ed8f2161b8358ff4c29c66
configure-options =
--with-mysql-source=${mariadb:location}__compile__/mariadb-${mariadb:version}
--with-mysql-config=${mariadb:location}/bin/mysql_config
......
component/percona-toolkit/buildout.cfg
View file @ 4b05a7da
......@@ -10,7 +10,7 @@ parts =
recipe = hexagonit.recipe.cmmi
depends =
${perl:version}
url = http://www.percona.com/redir/downloads/percona-toolkit/2.0.4/percona-toolkit-2.0.4.tar.gz
md5sum = df7dffcccb48d50f143849629228d4b4
url = http://www.percona.com/redir/downloads/percona-toolkit/2.1.3/percona-toolkit-2.1.3.tar.gz
md5sum = 6f8c4a0a0e43e467766cbdabaec9ef93
configure-command =
${perl:location}/bin/perl Makefile.PL
component/poppler/buildout.cfg
View file @ 4b05a7da
......@@ -13,8 +13,8 @@ extends =
[poppler]
recipe = hexagonit.recipe.cmmi
md5sum = 45dd2c16c8c4d1a39e830e45745c4e25
url = http://poppler.freedesktop.org/poppler-0.20.2.tar.gz
md5sum = 28c40266f374e1960a7bcead17d39f96
url = http://poppler.freedesktop.org/poppler-0.20.3.tar.gz
depends =
${libtiff:version}
configure-options =
......
component/slapos/buildout.cfg
View file @ 4b05a7da
# This file is used to install SlapOS Node itself.
[buildout]
extends =
../../stack/shacache-client.cfg
......@@ -112,26 +114,27 @@ scripts =
slapgrid-supervisord = slapos.grid.svcbackend:supervisord
slapproxy = slapos.proxy:main
slapos = slapos.entry:main
slapos-watchdog = slapos.grid.watchdog:main
[versions]
zc.buildout = 1.6.0-dev-SlapOS-006
zc.buildout = 1.6.0-dev-SlapOS-007
Jinja2 = 2.6
Werkzeug = 0.8.3
buildout-versions = 1.7
collective.recipe.template = 1.9
hexagonit.recipe.cmmi = 1.6
lxml = 3.0alpha2
meld3 = 0.6.8
lxml = 2.3.6
meld3 = 0.6.9
netaddr = 0.7.10
slapos.core = 0.28.6
slapos.core = 0.31.1
slapos.libnetworkcache = 0.13.2
xml-marshaller = 0.9.7
z3c.recipe.scripts = 1.0.1
zc.recipe.egg = 1.3.2
# Required by:
# slapos.core==0.28.6
# slapos.core==0.31.1
Flask = 0.9
# Required by:
......@@ -139,21 +142,21 @@ Flask = 0.9
hexagonit.recipe.download = 1.5.1
# Required by:
# slapos.core==0.28.6
# slapos.core==0.31.1
netifaces = 0.8
# Required by:
# slapos.core==0.28.6
# slapos.core==0.31.1
# slapos.libnetworkcache==0.13.2
# supervisor==3.0a12
# zc.buildout==1.6.0-dev-SlapOS-006
# supervisor==3.0b1
# zc.buildout==1.6.0-dev-SlapOS-007
# zope.interface==4.0.1
setuptools = 0.6c12dev-r88846
# Required by:
# slapos.core==0.28.6
supervisor = 3.0a12
# slapos.core==0.31.1
supervisor = 3.0b1
# Required by:
# slapos.core==0.28.6
# slapos.core==0.31.1
zope.interface = 4.0.1
component/sphinx/README 0 → 100644
View file @ 4b05a7da
This component is not used anymore - but might be revived within 4 months.
If not used within 6 months, feel free to delete this.
component/sphinx/buildout.cfg
View file @ 4b05a7da
......@@ -20,8 +20,8 @@ filename = sphinx-1.10-beta-snowball.patch
recipe = hexagonit.recipe.cmmi
depends =
${mariadb:version}
url = http://sphinxsearch.com/files/sphinx-2.0.2-beta.tar.gz
md5sum = fafe0f1a71d0ded32404c067eba7d0b3
url = http://sphinxsearch.com/files/sphinx-2.0.5-release.tar.gz
md5sum = e71fdb5b0c2911247d48fb30550b9584
configure-options =
--with-mysql
--with-mysql-includes=${mariadb:location}/include/mysql
......
setup.py
View file @ 4b05a7da
......@@ -2,7 +2,7 @@ from setuptools import setup, find_packages
import glob
import os
version = '0.67-dev'
version = '0.68.2-dev'
name = 'slapos.cookbook'
long_description = open("README.txt").read() + "\n" + \
open("CHANGES.txt").read() + "\n"
......@@ -36,10 +36,12 @@ setup(name=name,
'xml_marshaller', # need to communication with slapgrid
'zc.buildout', # plays with buildout
'zc.recipe.egg', # for scripts generation
'pytz', # for timezone database
],
zip_safe=True,
entry_points={
'zc.buildout': [
'addresiliency = slapos.recipe.addresiliency:Recipe',
'agent = slapos.recipe.agent:Recipe',
'apache.frontend = slapos.recipe.apache_frontend:Recipe',
'apachephp = slapos.recipe.apachephp:Recipe',
......@@ -63,6 +65,7 @@ setup(name=name,
'dropbear.add_authorized_key = slapos.recipe.dropbear:AddAuthorizedKey',
'dropbear.client = slapos.recipe.dropbear:Client',
'dropbear = slapos.recipe.dropbear:Recipe',
'dumpmdb = slapos.recipe.dumpmdb:Recipe',
'duplicity = slapos.recipe.duplicity:Recipe',
'egg_test = slapos.recipe.erp5_test:EggTestRecipe',
'equeue = slapos.recipe.equeue:Recipe',
......@@ -86,6 +89,7 @@ setup(name=name,
'gitinit = slapos.recipe.gitinit:Recipe',
'haproxy = slapos.recipe.haproxy:Recipe',
'helloworld = slapos.recipe.helloworld:Recipe',
'importmdb = slapos.recipe.importmdb:Recipe',
'java = slapos.recipe.java:Recipe',
'kumofs = slapos.recipe.kumofs:Recipe',
'kvm.frontend = slapos.recipe.kvm_frontend:Recipe',
......@@ -116,7 +120,7 @@ setup(name=name,
'publishurl = slapos.recipe.publishurl:Recipe',
'pwgen = slapos.recipe.pwgen:Recipe',
'pwgen.stable = slapos.recipe.pwgen:StablePasswordGeneratorRecipe',
'requestoptional = slapos.recipe.requestoptional:Recipe',
'requestoptional = slapos.recipe.request:RequestOptional',
'request = slapos.recipe.request:Recipe',
'seleniumrunner = slapos.recipe.seleniumrunner:Recipe',
'sheepdogtestbed = slapos.recipe.sheepdogtestbed:SheepDogTestBed',
......@@ -130,6 +134,7 @@ setup(name=name,
'slapmonitor = slapos.recipe.slapmonitor:Recipe',
'slapreport = slapos.recipe.slapreport:Recipe',
'slaprunner = slapos.recipe.slaprunner:Recipe',
'slaprunner.test = slapos.recipe.slaprunner:Test',
'softwaretype = slapos.recipe.softwaretype:Recipe',
'sphinx= slapos.recipe.sphinx:Recipe',
'sshkeys_authority.request = slapos.recipe.sshkeys_authority:Request',
......
slapos/recipe/requestoptional.py → slapos/recipe/addresiliency/__init__.py
View file @ 4b05a7da
......@@ -24,84 +24,58 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import logging
from slapos.recipe.librecipe import GenericSlapRecipe
from slapos import slap as slapmodule
import sys
import os
class Recipe(object):
""" Request a SlapOS instance. Won't fail if instance is not ready.
"""
def __init__(self, buildout, name, options):
self.logger = logging.getLogger(name)
slap = slapmodule.slap()
self.software_release_url = options['software-url']
self.name = options['name']
slap.initializeConnection(options['server-url'],
options.get('key-file'),
options.get('cert-file'),
)
computer_partition = slap.registerComputerPartition(
options['computer-id'], options['partition-id'])
self.request = computer_partition.request
self.isSlave = False
if 'slave' in options:
self.isSlave = options['slave'].lower() in ['y', 'yes', 'true', '1']
self.return_parameters = []
if 'return' in options:
self.return_parameters = [str(parameter).strip()
for parameter in options['return'].split()]
else:
self.logger.debug("No parameter to return to main instance."
"Be careful about that...")
software_type = 'RootSoftwareInstance'
if 'software-type' in options:
software_type = options['software-type']
filter_kw = {}
if 'sla' in options:
for sla_parameter in options['sla'].split():
filter_kw[sla_parameter] = options['sla-%s' % sla_parameter]
partition_parameter_kw = {}
if 'config' in options:
for config_parameter in options['config'].split():
partition_parameter_kw[config_parameter] = \
options['config-%s' % config_parameter]
self.instance = self.request(options['software-url'], software_type,
self.name, partition_parameter_kw=partition_parameter_kw,
filter_kw=filter_kw, shared=self.isSlave)
class Recipe(GenericSlapRecipe):
""" This class provides the installation of the resilience
script on the partition.
"""
self.failed = None
for param in self.return_parameters:
try:
options['connection-%s' % param] = str(
self.instance.getConnectionParameter(param))
except slapmodule.NotFoundError:
options['connection-%s' % param] = ''
if self.failed is None:
self.failed = param
def _install(self):
path_list = []
self_id = int(self.parameter_dict['number'])
ip = self.parameter_dict['ip-list'].split(' ')
print 'Creating bully script with ips : %s\n' % ip
slap_connection = self.buildout['slap-connection']
def install(self):
if self.failed is not None:
# Check instance status to know if instance has been deployed
try:
status = self.instance.getState()
except slapmodule.NotFoundError:
status = 'not ready yet'
except AttributeError:
status = 'unknown'
error_message = 'Connection parameter %s not found. '\
'Requested instance is currently %s. If this error persists, '\
'check status of this instance.' % (self.failed, status)
self.logger.error(error_message)
return []
path_conf = os.path.join(self.options['script'], 'conf.in')
path_bully = os.path.join(self.options['script'], self.parameter_dict['script'])
path_bully_new = os.path.join(self.options['script'], 'new.py')
path_run = os.path.join(self.options['run'], self.parameter_dict['wrapper'])
print 'paths: %s\n%s\n' % (path_run, path_bully)
bully_conf = dict(self_id=self_id,
ip_list=ip,
executable=sys.executable,
syspath=sys.path,
server_url=slap_connection['server-url'],
key_file=slap_connection.get('key-file'),
cert_file=slap_connection.get('cert-file'),
computer_id=slap_connection['computer-id'],
partition_id=slap_connection['partition-id'],
software=slap_connection['software-release-url'],
namebase=self.parameter_dict['namebase'],
confpath=path_conf)
try:
conf = self.createFile(path_conf,
self.substituteTemplate(
self.getTemplateFilename('conf.in.in'),
bully_conf))
path_list.append(conf)
script = self.createExecutable(path_bully,
self.substituteTemplate(
self.getTemplateFilename('bully.py.in'),
bully_conf))
path_list.append(script)
update = install
wrapper = self.createPythonScript(
path_run,
'slapos.recipe.librecipe.execute.execute',
[path_bully])
path_list.append(wrapper)
except IOError:
pass
return path_list
slapos/recipe/addresiliency/template/bully.py.in 0 → 100644
View file @ 4b05a7da
#!%(executable)s
import select
import socket
import threading
import time
import sys
sys.path[:] = %(syspath)s
from slapos import slap as slapmodule
port = 50000
size = 1024
wait = True
def loadConnectionInfos():
connectionInfos = {}
file = open('%(confpath)s', 'r')
params = file.read().split('\n')
file.close()
ip_list = [x.strip("' ") for x in params[0].strip('[],').split(',')]
connectionInfos['self_id'] = int(params[1])
connectionInfos['server_list'] = \
[(i, ip_list[i]) for i in range(len(ip_list))]
connectionInfos['self_ip'] = ip_list[connectionInfos['self_id']]
return connectionInfos
def rename_broken_and_stop():
try:
slap = slapmodule.slap()
slap.initializeConnection('%(server_url)s',
'%(key_file)s',
'%(cert_file)s')
computer_partition = slap.registerComputerPartition('%(computer_id)s',
'%(partition_id)s')
broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename('%(namebase)s0')
print 'renaming done\n'
except slapos.slap.slap.ServerError:
print 'Internal server error\n'
def election():
global wait
connection = loadConnectionInfos()
message = "%%s, %%s" %% (connection['self_id'], "Election")
victory = True
for (remote_id, addr) in connection['server_list']:
if remote_id > connection['self_id']:
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.send(message)
reply = s.recv(size)
if reply == "%%s, %%s" %% (remote_id, "Alive"):
victory = False
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
if victory:
wait = True
for (remote_id, addr) in connection['server_list']:
if remote_id < connection['self_id']:
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.send("%%s, %%s" %% (connection['self_id'], "Victory"))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
rename_broken_and_stop()
def failure_detect():
global wait
connection = loadConnectionInfos()
while True:
time.sleep(30)
if wait:
print 'waiting 30 minutes\n'
time.sleep(30 * 60)
wait = False
if not connection['server_list'][0]:
continue
(remote_id, addr) = connection['server_list'][0]
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((addr, port + remote_id))
s.close()
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
s.close()
election()
failure_detect_thread = threading.Thread(target=failure_detect)
failure_detect_thread.start()
connection = loadConnectionInfos()
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.bind((connection['self_ip'], port + connection['self_id']))
s.listen(5)
#election()
while True:
force_election = False
client, _ = s.accept()
client_message = client.recv(1024)
if client_message:
client_id, message = client_message.split(', ')
client_id = eval(client_id)
if message == "Victory":
wait = True
print "%%s wins" %% client_id
elif message == "Election":
print "%%s starts an election" %% client_id
if client_id < connection['self_id']:
client.send("%%s, %%s" %% (connection['self_id'], "Alive"))
force_election = True
client.close()
if force_election:
election()
slapos/recipe/addresiliency/template/bully_new.py.in 0 → 100644
View file @ 4b05a7da
#!%(executable)s
import socket
import time
import sys
import thread
import time
import os
sys.path[:] = %(syspath)s
from slapos import slap as slapmodule
port = 50000
size = 1024
def rename_broken_and_stop():
try:
slap = slapmodule.slap()
slap.initializeConnection('%(server_url)s',
'%(key_file)s',
'%(cert_file)s')
computer_partition = slap.registerComputerPartition('%(computer_id)s',
'%(partition_id)s')
broken = computer_partition.request('%(software)s', 'frozen', '%(namebase)s0')
broken.rename('broken-%%s' %% (time.strftime("%%d-%%b_%%H:%%M:%%S", time.gmtime())))
broken.stopped()
computer_partition.rename('%(namebase)s0')
print 'renaming done\n'
except slapos.slap.slap.ServerError:
print 'Internal server error\n'
## Leader is always number 0
class ResilientInstance(object):
def __init__(self, comm):
self.comm = comm
self.id = 0
self.state = 'normal'
self.halter = 0
self.nbComp = nbComp
self.inElection = False
self.alive = True
self.lastPing = time.clock()
self.mainCanal = self.comm.canal(['ping', 'halt',
'victory'])
self.okCanal = self.comm.canal(['ok'])
self.loadConnectionInfos()
def loadConnectionInfos(self):
file = open('%(confpath)s', 'r')
params = file.read().split('\n')
file.close()
self.nbComp = len([x.strip("' ") for x in params[0].strip('[],').split(',')])
new_id = int(params[1])
if self.id != new_id:
self.halter = new_id
self.id = new_id
## Needs to be changed to use the master
def aliveManagement(self):
while self.alive:
time.sleep(30*60)
if self.id == 0:
continue
self.comm.send('ping', 0)
message, sender = self.okCanal.get()
if message:
continue
self.election()
def listen(self):
while self.alive:
self.comm.recv()
def main(self):
while self.alive:
message, sender = self.mainCanal.get()
if message == 'ping':
self.comm.send('ok', sender)
elif message == 'halt':
self.state = 'waitingConfirm'
self.halter = sender
self.comm.send('ok', sender)
elif message == 'victory':
if int(sender) == int(self.halter) and self.state == 'waitingConfirm':
print '%s thinks %s is the leader\n' % (self.id, sender)
self.comm.send('ok', sender)
self.state = 'normal'
def election(self):
self.inElection = True
self.loadConnectionInfos()
#Check if I'm the highest instance alive
for higher in range(self.id + 1, self.nbComp):
self.comm.send('ping', higher)
message, sender = self.okCanal.get()
if message:
#print '%s is alive (%s)\n' % (higher, self.id)
self.inElection = False
return False
continue
if not self.alive:
return False
#I should be the new coordinator, halt those below me
print 'Should be ME : %s \n' % self.id
self.state = 'election'
self.halter = self.id
ups = []
for lower in range(self.id):
self.comm.send('halt', lower)
message, sender = self.okCanal.get()
if message:
ups.append(lower)
#Broadcast Victory
self.state = 'reorganization'
for up in ups:
self.comm.send('victory', up)
message, sender = self.okCanal.get()
if message:
continue
print 'Something is wrong... let\'s start over\n'
return self.election()
self.state = 'normal'
self.active = True
print '%s Is THE LEADER \n' % self.id
rename_broken_and_stop()
self.inElection = False
return True
class FilteredCanal(object):
def __init__(self, accept, timeout):
self.accept = accept
self.list = []
self.lock = thread.allocate_lock()
self.timeout = timeout
def append(self, message, sender):
if message in self.accept:
self.lock.acquire()
self.list.append([message, sender])
self.lock.release()
def get(self):
start = time.clock()
while (time.clock() - start < self.timeout):
self.lock.acquire()
if self.list:
self.lock.release()
val = self.list[0]
self.list = self.list[1:]
return val
self.lock.release()
return [None, None]
class Wrapper(object):
def __init__(self, timeout=20):
self.read_pipes = [os.fdopen(x) for x in read_pipes]
self.write_pipes = write_pipes
self.canals = []
self.ips = []
self.id = 0
self.timeout = timeout
self.getConnectionInfos()
self.socket = None
def getConnectionInfos(self):
file = open('%(confpath)s', 'r')
params = file.read().split('\n')
file.close()
self.ips = [x.strip("' ") for x in params[0].strip('[],').split(',')]
self.id = int(params[1])
def start(self):
self.getConnectionInfos()
self.socket = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
self.socket.bind((self.ips[self.id], port + self.id))
s.listen(5)
def send(self, message, number):
self.getConnectionInfos()
try:
s = socket.socket(socket.AF_INET6, socket.SOCK_STREAM)
s.connect((self.ips[number], port + number))
s.send(message + (' %s\n' % self.id))
except (socket.error, socket.herror, socket.gaierror, socket.timeout):
pass
finally:
s.close()
def canal(self, accept):
created = FilteredCanal(accept, self.timeout)
self.canals.append(created)
return created
def recv(self):
client, _ = s.accept()
client_message = client.recv(1024)
if client_message:
message, sender = client_message.split()
for canal in self.canals:
canal.append(message, sender)
wrapper = createWrapper(20)
computer = ResilientInstance(wrapper)
#idle waiting for connection infos
while computer.nbComp < 2 :
computer.loadConnectionInfos()
time.sleep(30)
print 'Starting\n'
computer.comm.start()
thread.start_new_thread(computer.listen, ())
thread.start_new_thread(computer.main, ())
thread.start_new_thread(computer.aliveManagement, ())
while True:
continue
slapos/recipe/apacheproxy/template/apache.in
View file @ 4b05a7da
......@@ -3,7 +3,6 @@
# Basic server configuration
PidFile "%(pid_file)s"
LockFile "%(lock_file)s"
Listen %(ip)s:%(port)s
ServerAdmin someone@email
DefaultType text/plain
......@@ -22,13 +21,15 @@ CustomLog "%(access_log)s" common
<Directory />
Options FollowSymLinks
AllowOverride None
Order deny,allow
Deny from all
Require all denied
</Directory>
ProxyPass / %(backend_url)s
# List of modules
LoadModule unixd_module modules/mod_unixd.so
LoadModule access_compat_module modules/mod_access_compat.so
LoadModule authz_core_module modules/mod_authz_core.so
LoadModule authz_host_module modules/mod_authz_host.so
LoadModule log_config_module modules/mod_log_config.so
LoadModule setenvif_module modules/mod_setenvif.so
......
slapos/recipe/check_url_available/template/check_url.in
View file @ 4b05a7da
......@@ -9,11 +9,22 @@ if [ -z $URL ]; then
exit 3
fi
CODE=$(%(curl_path)s -k -sL $URL -w %%{http_code} -o /dev/null)
CODE=$(%(curl_path)s -g -k -sL $URL -w %%{http_code} -o /dev/null)
if [ $? -eq 3 ]; then
echo "URL malformed: $URL." >&2
exit 1
fi
if [ $? -eq 7 ]; then
echo "Failed to connect to host: $URL." >&2
exit 1
fi
if [ ! $CODE ]; then
echo "$URL is not available (server not reachable)." >&2
exit 1
fi
if [ $CODE -eq 000 ]; then
echo "$URL is not available (server not reachable)." >&2
......
slapos/recipe/dropbear.py
View file @ 4b05a7da
......@@ -167,6 +167,7 @@ class AddAuthorizedKey(GenericBaseRecipe):
path_list.append(ssh)
authorized_keys = AuthorizedKeysFile(os.path.join(ssh, 'authorized_keys'))
authorized_keys.append(self.options['key'])
for key in self.options['key'].split(' '):
authorized_keys.append(key)
return path_list
slapos/recipe/generic_mysql/__init__.py
View file @ 4b05a7da
......@@ -154,26 +154,38 @@ class Recipe(GenericBaseRecipe):
# TODO: move to a separate recipe (ack'ed by Cedric)
# percona toolkit (formerly known as maatkit) installation
for pt_script_name in (
'pt-align',
'pt-archiver',
'pt-collect',
'pt-config-diff',
'pt-deadlock-logger',
'pt-diskstats',
'pt-duplicate-key-checker',
'pt-fifo-split',
'pt-find',
'pt-fingerprint',
'pt-fk-error-logger',
'pt-heartbeat',
'pt-index-usage',
'pt-ioprofile',
'pt-kill',
'pt-log-player',
'pt-mext',
'pt-mysql-summary',
'pt-online-schema-change',
'pt-pmp',
'pt-query-advisor',
'pt-query-digest',
'pt-show-grants',
'pt-sift',
'pt-slave-delay',
'pt-slave-find',
'pt-slave-restart',
'pt-stalk',
'pt-summary',
'pt-table-checksum',
'pt-table-sync',
'pt-table-usage',
'pt-tcp-model',
'pt-trend',
'pt-upgrade',
......
slapos/recipe/generic_mysql/mysql.py
View file @ 4b05a7da
......@@ -2,6 +2,7 @@ import os
import subprocess
import time
import sys
import pytz
def runMysql(args):
......@@ -41,32 +42,50 @@ def updateMysql(args):
sleep = 30
is_succeed = False
while True:
if not is_succeed:
mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql_upgrade.communicate()[0]
if mysql_upgrade.returncode is None:
mysql_upgrade.kill()
if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
print 'Sleeping for %ss and retrying' % sleep
mysql_upgrade_list = [conf['mysql_upgrade_binary'], '--no-defaults', '--user=root', '--socket=%s' % conf['socket']]
mysql_upgrade = subprocess.Popen(mysql_upgrade_list, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql_upgrade.communicate()[0]
if mysql_upgrade.returncode is None:
mysql_upgrade.kill()
if mysql_upgrade.returncode != 0 and not 'is already upgraded' in result:
print "Command %r failed with result:\n%s" % (mysql_upgrade_list, result)
else:
if mysql_upgrade.returncode == 0:
print "MySQL database upgraded with result:\n%s" % result
else:
if mysql_upgrade.returncode == 0:
print "MySQL database upgraded with result:\n%s" % result
else:
print "No need to upgrade MySQL database"
mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']]
mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql.communicate(conf['mysql_script'])[0]
if mysql.returncode is None:
mysql.kill()
print "No need to upgrade MySQL database"
mysql_list = [conf['mysql_binary'].strip(), '--no-defaults', '-B', '--user=root', '--socket=%s' % conf['socket']]
mysql = subprocess.Popen(mysql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql.communicate(conf['mysql_script'])[0]
if mysql.returncode is None:
mysql.kill()
if mysql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_list, result)
else:
# import timezone database
mysql_tzinfo_to_sql_binary = os.path.join(
os.path.dirname(conf['mysql_binary'].strip()), 'mysql_tzinfo_to_sql')
zoneinfo_directory = '%s/zoneinfo' % os.path.dirname(pytz.__file__)
mysql_tzinfo_to_sql_list = [mysql_tzinfo_to_sql_binary, zoneinfo_directory]
mysql_tzinfo_to_sql = subprocess.Popen(mysql_tzinfo_to_sql_list, stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.PIPE)
timezone_sql = mysql_tzinfo_to_sql.communicate()[0]
if mysql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_list, result)
print 'Sleeping for %ss and retrying' % sleep
print 'Command %r failed with:\n%s' % (mysql_tzinfo_to_sql_list, result)
else:
mysql = subprocess.Popen(mysql_list + ['mysql',], stdin=subprocess.PIPE,
stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
result = mysql.communicate(timezone_sql)[0]
if mysql.returncode is None:
mysql.kill()
if mysql.returncode != 0:
print 'Command %r failed with:\n%s' % (mysql_list, result)
is_succeed = True
print 'SlapOS initialisation script succesfully applied on database.'
if is_succeed:
print 'SlapOS initialisation script succesfully applied on database.'
break
print 'Sleeping for %ss and retrying' % sleep
sys.stdout.flush()
sys.stderr.flush()
time.sleep(sleep)
slapos/recipe/generic_mysql/template/my.cnf.in
View file @ 4b05a7da
......@@ -24,7 +24,7 @@ long_query_time = 1
max_allowed_packet = 128M
query_cache_size = 32M
plugin-load = ha_mroonga.so;ha_sphinx.so;handlersocket.so
plugin-load = ha_mroonga.so;handlersocket.so
# By default only 100 connections are allowed, when using zeo
# we may have much more connections
......
slapos/recipe/haproxy/__init__.py
View file @ 4b05a7da
......@@ -73,7 +73,11 @@ class Recipe(GenericBaseRecipe):
server_snippet = ""
i = 0
name = self.options['name']
for address in self.options['backend-list'].split():
backend_list = self.options['backend-list']
if isinstance(backend_list, str):
# BBB
backend_list = backend_list.split()
for address in backend_list:
i += 1
server_snippet += self.substituteTemplate(
snippet_filename, dict(
......
slapos/recipe/lamp/template/my.cnf.in
View file @ 4b05a7da
......@@ -21,7 +21,7 @@ long_query_time = 5
max_allowed_packet = 128M
query_cache_size = 32M
plugin-load = ha_mroonga.so;ha_sphinx.so
plugin-load = ha_mroonga.so
# The following are important to configure and depend a lot on to the size of
# your database and the available resources.
......
slapos/recipe/mysql/__init__.py
View file @ 4b05a7da
......@@ -31,6 +31,7 @@ import subprocess
from slapos.recipe.librecipe import GenericBaseRecipe
from slapos.recipe.librecipe import filehash
class Recipe(GenericBaseRecipe):
def _options(self, options):
......
slapos/recipe/mysql/template/my.cnf.in
View file @ 4b05a7da
......@@ -19,7 +19,7 @@ long_query_time = 5
max_allowed_packet = 128M
query_cache_size = 32M
plugin-load = ha_mroonga.so;ha_sphinx.so
plugin-load = ha_mroonga.so
# The following are important to configure and depend a lot on to the size of
# your database and the available resources.
......
slapos/recipe/pbs.py
View file @ 4b05a7da
......@@ -213,6 +213,7 @@ class Recipe(GenericSlapRecipe, Notify, Callback):
for slave in slaves:
path_list.extend(self.add_slave(slave, known_hosts))
else:
command = [self.options['rdiffbackup-binary']]
self.logger.info("Server mode")
......
slapos/recipe/request.py
View file @ 4b05a7da
......@@ -27,6 +27,9 @@
import logging
from slapos import slap as slapmodule
import slapos.recipe.librecipe.generic as librecipe
DEFAULT_SOFTWARE_TYPE = 'RootSoftwareInstance'
class Recipe(object):
"""
......@@ -97,17 +100,15 @@ class Recipe(object):
request = slap.registerComputerPartition(
options['computer-id'], options['partition-id']).request
isSlave = options.get('slave', '').lower() in ['y', 'yes', 'true', '1']
return_parameters = []
if 'return' in options:
return_parameters = [str(parameter).strip()
for parameter in options['return'].split()]
for parameter in options['return'].split()]
else:
self.logger.debug("No parameter to return to main instance."
"Be careful about that...")
software_type = options.get('software-type', 'RootInstanceSoftware')
software_type = options.get('software-type', DEFAULT_SOFTWARE_TYPE)
filter_kw = {}
if 'sla' in options:
......@@ -120,6 +121,8 @@ class Recipe(object):
partition_parameter_kw[config_parameter] = \
options['config-%s' % config_parameter]
isSlave = options.get('slave', '').lower() in \
librecipe.GenericBaseRecipe.TRUE_VALUES
self.instance = instance = request(software_url, software_type,
name, partition_parameter_kw=partition_parameter_kw,
filter_kw=filter_kw, shared=isSlave)
......@@ -128,7 +131,7 @@ class Recipe(object):
try:
options['connection-%s' % param] = str(
instance.getConnectionParameter(param))
except slapmodule.NotFoundError:
except (slapmodule.NotFoundError, slapmodule.ServerError):
options['connection-%s' % param] = ''
if self.failed is None:
self.failed = param
......@@ -137,9 +140,12 @@ class Recipe(object):
if self.failed is not None:
# Check instance status to know if instance has been deployed
try:
status = self.instance.getState()
except slapmodule.NotFoundError:
status = 'not ready yet, please try again'
if self.instance.getComputerId() is not None:
status = self.instance.getState()
else:
status = 'not ready yet'
except (slapmodule.NotFoundError, slapmodule.ServerError):
status = 'not ready yet'
except AttributeError:
status = 'unknown'
error_message = 'Connection parameter %s not found. '\
......@@ -150,3 +156,28 @@ class Recipe(object):
return []
update = install
class RequestOptional(Recipe):
"""
Request a SlapOS instance. Won't fail if instance is not ready.
Same as slapos.cookbook:request, but won't raise in case of problem.
"""
def install(self):
if self.failed is not None:
# Check instance status to know if instance has been deployed
try:
if self.instance.getComputerId() is not None:
status = self.instance.getState()
else:
status = 'not ready yet'
except (slapmodule.NotFoundError, slapmodule.ServerError):
status = 'not ready yet'
except AttributeError:
status = 'unknown'
error_message = 'Connection parameter %s not found. '\
'Requested instance is currently %s. If this error persists, '\
'check status of this instance.' % (self.failed, status)
self.logger.warning(error_message)
return []
update = install
slapos/recipe/slaprunner/__init__.py
View file @ 4b05a7da
......@@ -39,6 +39,7 @@ class Recipe(GenericBaseRecipe):
self.instance_directory = options['instance-directory'].strip()
self.partition_amount = options['partition-amount'].strip()
self.cloud9_url = options['cloud9-url'].strip()
self.log_file = os.path.join(options['log_dir'].strip(), 'slaprunner.log')
# Set slaprunner access URL
options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
......@@ -79,10 +80,12 @@ class Recipe(GenericBaseRecipe):
path_list.append(config_file)
environment = dict(
PATH=os.path.dirname(self.options['git-binary']) + ':' + os.environ['PATH'],
PATH=os.path.dirname(
self.options['git-binary']) + ':' + os.environ['PATH'],
GIT_SSH=self.options['ssh_client']
)
launch_args = [self.options['slaprunner'].strip(), config_file]
launch_args = [self.options['slaprunner'].strip(), config_file,
'--log_file', self.log_file]
if self.optionIsTrue('debug', default=False):
launch_args.append('--debug')
......@@ -93,3 +96,69 @@ class Recipe(GenericBaseRecipe):
path_list.append(wrapper)
return path_list
class Test(GenericBaseRecipe):
def _options(self, options):
self.ipv4 = options['ipv4'].strip()
self.ipv6 = options['ipv6'].strip()
self.proxy_port = options['proxy_port'].strip()
self.runner_port = options['runner_port'].strip()
self.workdir = options['working-directory'].strip()
self.software_directory = options['software-directory'].strip()
self.instance_directory = options['instance-directory'].strip()
self.partition_amount = options['partition-amount'].strip()
self.cloud9_url = options['cloud9-url'].strip()
# Set slaprunner access URL
options['access-url'] = 'http://[%s]:%s' % (self.ipv6, self.runner_port)
def install(self):
path_list = []
configuration = dict(
software_root=self.software_directory,
instance_root=self.instance_directory,
master_url='http://%s:%s/' % (self.ipv4, self.proxy_port),
computer_id='slaprunner',
partition_amount=self.partition_amount,
slapgrid_sr=self.options['slapgrid_sr'],
slapgrid_cp=self.options['slapgrid_cp'],
slapproxy=self.options['slapproxy'],
supervisor=self.options['supervisor'],
supervisord_config=os.path.join(self.instance_directory, 'etc',
'supervisord.conf'),
runner_workdir=self.workdir,
etc_dir=self.options['etc_dir'],
runner_host=self.ipv6,
runner_port=self.runner_port,
ipv4_address=self.ipv4,
ipv6_address=self.ipv6,
proxy_host=self.ipv4,
proxy_port=self.proxy_port,
proxy_database=os.path.join(self.workdir, 'proxy.db'),
git=self.options['git-binary'],
ssh_client=self.options['ssh_client'],
public_key=self.options['public_key'],
private_key=self.options['private_key'],
cloud9_url=self.cloud9_url
)
config_file = self.createFile(self.options['slapos.cfg'],
self.substituteTemplate(self.getTemplateFilename('slapos.cfg.in'),
configuration))
path_list.append(config_file)
environment = dict(
PATH=os.path.dirname(
self.options['git-binary']) + ':' + os.environ['PATH'],
GIT_SSH=self.options['ssh_client'],
CONFIG_FILE_PATH=config_file
)
launch_args = [self.options['slaprunnertest'].strip()]
wrapper = self.createPythonScript(self.options['wrapper'],
'slapos.recipe.librecipe.execute.executee',
(launch_args, environment)
)
path_list.append(wrapper)
return path_list
slapos/recipe/slaprunner/execute.py deleted 100644 → 0
View file @ fb52bb07
import os
def execute(args):
"""Portable execution with process replacement"""
if args.get("path", None):
os.environ['PATH'] = args["path"]
os.execv(args["launch_args"][0], args["launch_args"])
slapos/recipe/softwaretype.py
View file @ 4b05a7da
......@@ -87,6 +87,8 @@ class Recipe:
computer_partition_id)
self.parameter_dict = self.computer_partition.getInstanceParameterDict()
software_type = self.parameter_dict['slap_software_type']
self.logger.info('Deploying instance with software type %s' % \
software_type)
if software_type not in self.options:
if 'default' in self.options:
......
slapos/recipe/tidstorage/__init__.py
View file @ 4b05a7da
......@@ -29,13 +29,21 @@ from slapos.recipe.librecipe import GenericBaseRecipe
class Recipe(GenericBaseRecipe):
def install(self):
path_list = []
a = path_list.append
configuration_file = self.createFile(self.options['configuration-path'], self.substituteTemplate(self.getTemplateFilename('tidstorage.py.in'), self.options))
a(configuration_file)
tidstorage_wrapper = self.createPythonScript(self.options['tidstorage-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstoraged-binary'], '--nofork', '--config', configuration_file])
a(tidstorage_wrapper)
repozo_wrapper = self.createPythonScript(self.options['repozo-wrapper'], 'slapos.recipe.librecipe.execute.execute', [self.options['tidstorage-repozo-binary'], '--config', configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
a(repozo_wrapper)
configuration_file = self.createFile(
self.options['configuration-path'],
self.substituteTemplate(
self.getTemplateFilename('tidstorage.py.in'), self.options))
return path_list
tidstorage_wrapper = self.createPythonScript(
self.options['tidstorage-wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['tidstoraged-binary'], '--nofork', '--config',
configuration_file])
repozo_wrapper = self.createPythonScript(
self.options['repozo-wrapper'],
'slapos.recipe.librecipe.execute.execute',
[self.options['tidstorage-repozo-binary'], '--config',
configuration_file, '--repozo', self.options['repozo-binary'], '-z'])
return [configuration_file, tidstorage_wrapper, repozo_wrapper]
slapos/recipe/tidstorage/template/tidstorage.py.in
View file @ 4b05a7da
known_tid_storage_identifier_dict = %(known-tid-storage-identifier-dict)s
base_url = '%(base-url)s'
address = '%(ip)s'
port = %(port)s
......
slapos/recipe/xwiki/template/my.cnf.in
View file @ 4b05a7da
......@@ -21,7 +21,7 @@ long_query_time = 5
max_allowed_packet = 128M
query_cache_size = 32M
plugin-load = ha_mroonga.so;ha_sphinx.so
plugin-load = ha_mroonga.so
# The following are important to configure and depend a lot on to the size of
# your database and the available resources.
......
software/slaprunner/instance-runner.cfg
View file @ 4b05a7da
......@@ -2,6 +2,7 @@
parts =
cloud9
slaprunner
test-runner
sshkeys-dropbear
dropbear-server-add-authorized-key
sshkeys-authority
......@@ -11,7 +12,7 @@ parts =
cloud9-promise
dropbear-promise
eggs-directory = ${buildout:eggs-directory}
develop-eggs-directory = ${buildout:develop-eggs-directory}
offline = true
......@@ -38,13 +39,18 @@ services = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
test = $${rootdirectory:etc}/test/
[runnerdirectory]
recipe = slapos.cookbook:mkdirectory
home = $${rootdirectory:srv}/runner/
test = $${rootdirectory:srv}/test/
project = $${:home}/project
software-root = $${:home}/software
instance-root = $${:home}/instance
project-test = $${:test}/project
software-test = $${:test}/software
instance-test = $${:test}/instance
#Create password recovery code for slaprunner
[recovery-code]
......@@ -85,9 +91,10 @@ ipv4 = $${slap-network-information:local-ipv4}
ipv6 = $${slap-network-information:global-ipv6}
proxy_port = 50000
runner_port = 50000
partition-amount = 7
partition-amount = $${slap-parameter:instance-amount}
cloud9-url = $${cloud9:access-url}
wrapper = $${basedirectory:services}/slaprunner
debug = False
# Deploy dropbear (minimalist ssh server)
[sshkeys-directory]
......@@ -95,6 +102,18 @@ recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/
keys = $${directory:sshkeys}/keys/
[test-runner]
<= slaprunner
recipe = slapos.cookbook:slaprunner.test
slaprunnertest = ${buildout:directory}/bin/slaprunnertest
slapos.cfg = $${rootdirectory:etc}/slapos-test.cfg
working-directory = $${runnerdirectory:test}
project-directory = $${runnerdirectory:project-test}
software-directory = $${runnerdirectory:software-test}
instance-directory = $${runnerdirectory:instance-test}
wrapper = $${rootdirectory:bin}/runTestSuite
etc_dir = $${basedirectory:test}
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
......@@ -182,3 +201,5 @@ port = $${dropbear-server:port}
[slap-parameter]
# Default value if no ssh key is specified
authorized-key =
# Default value of instances number in slaprunner
instance-amount = 10
software/slaprunner/software.cfg
View file @ 4b05a7da
......@@ -25,7 +25,7 @@ md5sum = 5307e4200f044ae57b504ad68444491c
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg
md5sum = b57ebff565595fa92cd7d5adc6a22c7e
md5sum = bd1c73c149b4adf2fbab80bc29c782c8
mode = 0644
[eggs]
......@@ -54,44 +54,48 @@ signature-certificate-list =
-----END CERTIFICATE-----
[versions]
# Forced stable version of lxml - 3.0 alpha keeps appearing/diseappearing on
# Pypi
lxml = 2.3.6
Flask-Auth = 0.8
Jinja2 = 2.6
Werkzeug = 0.8.3
apache-libcloud = 0.11.1
apache-libcloud = 0.11.3
async = 0.6.1
buildout-versions = 1.7
gitdb = 0.5.4
hexagonit.recipe.cmmi = 1.6
meld3 = 0.6.8
meld3 = 0.6.9
plone.recipe.command = 1.1
pycrypto = 2.6
slapos.cookbook = 0.64.2
slapos.cookbook = 0.68.1
slapos.libnetworkcache = 0.13.2
slapos.recipe.build = 0.11
slapos.recipe.build = 0.11.5
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.30
slapos.toolbox = 0.31.1
smmap = 0.8.2
z3c.recipe.scripts = 1.0.1
# Required by:
# slapos.core==0.28.5
# slapos.toolbox==0.30
# slapos.core==0.31.2
# slapos.toolbox==0.31.1
Flask = 0.9
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31.1
GitPython = 0.3.2.RC1
# Required by:
# slapos.cookbook==0.64.2
# slapos.cookbook==0.68
PyXML = 0.8.4
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31.1
atomize = 0.1.1
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31.1
feedparser = 5.1.2
# Required by:
......@@ -99,58 +103,56 @@ feedparser = 5.1.2
hexagonit.recipe.download = 1.5.1
# Required by:
# slapos.cookbook==0.64.2
# slapos.cookbook==0.68
inotifyx = 0.2.0
# Required by:
# slapos.core==0.28.5
# slapos.toolbox==0.30
# xml-marshaller==0.9.7
lxml = 3.0alpha2
# Required by:
# slapos.cookbook==0.64.2
netaddr = 0.7.9
# slapos.cookbook==0.68
netaddr = 0.7.10
# Required by:
# slapos.core==0.28.5
# slapos.core==0.31.2
netifaces = 0.8
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31.1
paramiko = 1.7.7.2
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31.1
psutil = 0.6.1
# Required by:
# slapos.cookbook==0.64.2
# slapos.core==0.28.5
# slapos.cookbook==0.68
pytz = 2012f
# Required by:
# slapos.cookbook==0.68
# slapos.core==0.31.2
# slapos.libnetworkcache==0.13.2
# slapos.toolbox==0.30
# supervisor==3.0a12
# slapos.toolbox==0.31.1
# supervisor==3.0b1
# zc.buildout==1.6.0-dev-SlapOS-007
# zc.recipe.egg==1.3.2
# zope.interface==4.0.1
setuptools = 0.6c12dev-r88846
# Required by:
# slapos.cookbook==0.64.2
slapos.core = 0.28.5
# slapos.cookbook==0.68
slapos.core = 0.31.2
# Required by:
# slapos.core==0.28.5
supervisor = 3.0a12
# slapos.core==0.31.2
supervisor = 3.0b1
# Required by:
# slapos.cookbook==0.64.2
# slapos.cookbook==0.68
xml-marshaller = 0.9.7
# Required by:
# slapos.cookbook==0.64.2
# slapos.cookbook==0.68
zc.recipe.egg = 1.3.2
# Required by:
# slapos.core==0.28.5
# slapos.core==0.31.2
zope.interface = 4.0.1
stack/erp5/buildout.cfg
View file @ 4b05a7da
......@@ -17,7 +17,7 @@ find-links =
extends =
# Exact version of Zope
http://svn.zope.org/repos/main/Zope/tags/2.12.23/versions.cfg
http://svn.zope.org/repos/main/Zope/tags/2.12.25/versions.cfg
../../stack/slapos.cfg
../../component/logrotate/buildout.cfg
../../component/dcron/buildout.cfg
......@@ -48,7 +48,6 @@ extends =
../../component/python-2.7/buildout.cfg
../../component/python-ldap-python/buildout.cfg
../../component/rdiff-backup/buildout.cfg
../../component/sphinx/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/subversion/buildout.cfg
../../component/tesseract/buildout.cfg
......@@ -85,7 +84,6 @@ parts =
ghostscript
mariadb
mroonga-mariadb
sphinx
imagemagick
libdmtx
dmtx-utils
......@@ -186,7 +184,7 @@ context =
[template-mariadb]
< = template-jinja2-base
filename = instance-mariadb.cfg
md5sum = 0e8e9bfc8c8a5f1bd71f5c4f346ef68b
md5sum = 867bd8e3c4f56db2aab2fcad59f2efd8
extra-context =
key coreutils_location coreutils:location
key dcron_location dcron:location
......@@ -200,13 +198,6 @@ extra-context =
key sed_location sed:location
key xtrabackup_location xtrabackup:location
[template-sphinx]
< = template-jinja2-base
filename = instance-sphinx.cfg
md5sum = e3975afd5220f251d6b6e7024a2ca08b
extra-context =
key sphinx_location sphinx:location
[template-zope]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-zope.cfg.in
......@@ -227,7 +218,7 @@ extra-context =
[template-tidstorage]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-tidstorage.cfg.in
md5sum = 520e39f2612a045b42c0ad914bfdc5a9
md5sum = 1f3e7cfc719d3c2eee4860678f6a67df
mode = 640
[template-cloudooo]
......@@ -239,7 +230,7 @@ mode = 640
[template-zope-conf]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/zope.conf.in
md5sum = 599e004c18fd3461aa846ae75bd112f2
md5sum = bc2154161a1d5baddc4ed4dfaaf94fbe
mode = 640
[template]
......@@ -247,7 +238,7 @@ mode = 640
# XXX: "template.cfg" is hardcoded in instanciation recipe
filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 26d26c357053af48602b5110120bd085
md5sum = 700b8729a4d9a8423ab2a99de486cc31
extra-context =
key apache_location apache:location
key aspell_location aspell:location
......@@ -292,7 +283,6 @@ extra-context =
key template_kumofs template-kumofs:rendered
key template_mariadb template-mariadb:rendered
key template_memcached template-memcached:rendered
key template_sphinx template-sphinx:rendered
key template_tidstorage template-tidstorage:target
key template_varnish template-varnish:target
key template_zope template-zope:target
......@@ -320,7 +310,7 @@ mode = 640
[template-erp5-cluster]
< = template-jinja2-base
filename = instance-erp5-cluster.cfg
md5sum = 3051f3fea39d7409c417dcac26027b56
md5sum = b4a9688ca7983fac5dd573e66c6e4263
extra-context =
key local_bt5_repository local-bt5-repository:list
......@@ -473,9 +463,10 @@ eggs =
huBarcode
qrcode
# Zope 2.12 with patched acquisition
# Zope
ZODB3
Zope2
# Zope acquisition patch
Acquisition
# Other Zope 2 packages
......@@ -574,7 +565,7 @@ pysvn = 1.7.4nxd006
# CMF 2.3 requries Zope 2.13.
Products.CMFCalendar = 2.2.2
Products.CMFCore = 2.2.5
Products.CMFCore = 2.2.6
Products.CMFDefault = 2.2.2
Products.CMFTopic = 2.2.1
Products.CMFUid = 2.2.1
......@@ -583,7 +574,7 @@ Products.CMFUid = 2.2.1
slapos.cookbook =
# Pinned versions
Flask = 0.8
Flask = 0.9
GitPython = 0.3.2.RC1
MySQL-python = 1.2.3
PIL = 1.1.7
......@@ -591,68 +582,63 @@ Paste = 1.7.5.1
PasteDeploy = 1.5.0
PasteScript = 1.7.5
Products.CMFActionIcons = 2.1.3
Products.CMFCalendar = 2.2.2
Products.CMFCore = 2.2.5
Products.CMFDefault = 2.2.2
Products.CMFTopic = 2.2.1
Products.CMFUid = 2.2.1
Products.DCWorkflowGraph = 0.4.1
Products.ExternalEditor = 1.1.0
Products.GenericSetup = 1.7.1
Products.GenericSetup = 1.7.2
Products.MimetypesRegistry = 2.0.3
Products.PluggableAuthService = 1.7.7
Products.PluginRegistry = 1.3b1
Products.TIDStorage = 5.4.7.dev-r45842
Products.PluggableAuthService = 1.8.0
Products.PluginRegistry = 1.3
Products.TIDStorage = 5.4.8
Products.Zelenium = 1.0.3
StructuredText = 2.11.1
WSGIUtils = 0.7
Werkzeug = 0.8.3
apache-libcloud = 0.9.1
argparse = 1.1
apache-libcloud = 0.11.1
argparse = 1.2.1
async = 0.6.1
atomize = 0.1.1
buildout-versions = 1.7
cElementTree = 1.0.5-20051216
chardet = 1.0.1
coverage = 3.5.1
chardet = 1.1
coverage = 3.5.2
csp-eventlet = 0.7.0
elementtree = 1.2.7-20070827-preview
erp5.recipe.cmmiforcei686 = 0.1.3
erp5.util = 0.4.6
erp5.util = 0.4.7
erp5diff = 0.8.1.5
eventlet = 0.9.16
feedparser = 5.1.1
eventlet = 0.9.17
feedparser = 5.1.2
five.localsitemanager = 2.0.5
fpconst = 0.7.2
gitdb = 0.5.4
greenlet = 0.3.4
hexagonit.recipe.cmmi = 1.5.0
http-parser = 0.7.5
greenlet = 0.4.0
hexagonit.recipe.cmmi = 1.6
http-parser = 0.7.8
huBarcode = 0.63
inotifyx = 0.2.0
ipdb = 0.6.1
ipython = 0.12
ipdb = 0.7
ipython = 0.13
meld3 = 0.6.8
netaddr = 0.7.6
netaddr = 0.7.9
netifaces = 0.8
ordereddict = 1.1
paramiko = 1.7.7.1
paramiko = 1.7.7.2
plone.recipe.command = 1.1
ply = 3.4
polib = 1.0.0
psutil = 0.4.1
python-ldap = 2.4.9
python-magic = 0.4.2
psutil = 0.6.1
python-ldap = 2.4.10
python-magic = 0.4.3
python-memcached = 1.47
restkit = 4.1.2
restkit = 4.2.0
rtjp-eventlet = 0.3.2
setuptools = 0.6c12dev-r88846
slapos.core = 0.24
slapos.recipe.build = 0.7
slapos.recipe.template = 2.4
slapos.toolbox = 0.23
slapos.core = 0.28
slapos.recipe.build = 0.10
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.29
smmap = 0.8.2
socketpool = 0.3.0
socketpool = 0.5.2
supervisor = 3.0a12
threadframe = 0.2
timerserver = 2.0.2
......@@ -661,3 +647,6 @@ uuid = 1.30
validictory = 0.8.3
xml-marshaller = 0.9.7
xupdate-processor = 0.4
mr.developer = 1.21
qrcode = 2.4.1
requests = 0.13.2
stack/erp5/instance-erp5-cluster.cfg.in
View file @ 4b05a7da
......@@ -24,13 +24,6 @@ cert-file = ${slap-connection:cert-file}
computer-id = ${slap-connection:computer-id}
partition-id = ${slap-connection:partition-id}
[request-sphinx]
<=request-common
name = Sphinx Search Engine
software-type = sphinx
sla-computer_guid = ${slap-parameter:sphinx-computer-guid}
return = url-sphinx url-sphinx-sql
[request-mariadb]
<=request-common
name = MariaDB DataBase
......@@ -67,8 +60,6 @@ config =
memcached-url
cloudooo-url
kumofs-url
sphinx-url-sphinx
sphinx-url-sphinx-sql
smtp-url
bt5
bt5-repository-url
......@@ -77,8 +68,6 @@ config-mysql-url = ${request-mariadb:connection-url}
config-memcached-url = ${request-memcached:connection-url}
config-cloudooo-url = ${request-cloudooo:connection-url}
config-kumofs-url = ${request-kumofs:connection-url}
config-sphinx-url-sphinx = ${request-sphinx:connection-url-sphinx}
config-sphinx-url-sphinx-sql = ${request-sphinx:connection-url-sphinx-sql}
config-bt5 = ${slap-parameter:bt5}
config-bt5-repository-url = ${slap-parameter:bt5-repository-url}
config-smtp-url = ${slap-parameter:smtp-url}
......@@ -97,7 +86,6 @@ sla-computer_guid = ${slap-parameter:varnish-computer-guid}
[slap-parameter]
# Default value if no computer_guid is specified for each type
sphinx-computer-guid = ${slap-connection:computer-id}
mariadb-computer-guid = ${slap-connection:computer-id}
cloudooo-computer-guid = ${slap-connection:computer-id}
memcached-computer-guid = ${slap-connection:computer-id}
......
stack/erp5/instance-mariadb.cfg.in
View file @ 4b05a7da
......@@ -30,7 +30,7 @@ recipe = slapos.cookbook:generic.mysql
# Options
user = user
parallel-test-database-amount = 100
parallel-test-database-amount = ${slap-parameter:test-database-amount}
port = 45678
ip = ${slap-network-information:local-ipv4}
database = erp5
......@@ -178,3 +178,6 @@ recipe = slapos.cookbook:check_port_listening
path = ${basedirectory:promise}/mariadb
hostname = ${mariadb-instance:ip}
port = ${mariadb-instance:port}
[slap-parameter]
test-database-amount = 100
stack/erp5/instance-sphinx.cfg.in deleted 100644 → 0
View file @ fb52bb07
[buildout]
parts =
publish-sphinx-url
sphinxd-instance
eggs-directory = {{ eggs_directory }}
develop-eggs-directory = {{ develop_eggs_directory }}
offline = true
[publish-sphinx-url]
recipe = slapos.cookbook:publish
url-sphinx = sphinx://${sphinxd-instance:ip}:${sphinxd-instance:sphinx-port}
url-sphinx-sql = mysql://${sphinxd-instance:ip}:${sphinxd-instance:sql-port}
[sphinxd-instance]
recipe = slapos.cookbook:sphinx
data-directory = ${directory:sphinx-data}
configuration-file = ${rootdirectory:etc}/sphinx.conf
searchd-log = ${basedirectory:log}/sphinx-searchd.log
query-log = ${basedirectory:log}/sphinx-query.log
pid = ${basedirectory:run}/sphinx-searchd.pid
ip = ${slap-network-information:local-ipv4}
sphinx-port = 9312
sql-port = 9306
wrapper = ${basedirectory:services}/sphinxd
sphinx-searchd-binary = {{ sphinx_location }}/bin/searchd
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
var = ${buildout:directory}/var
srv = ${buildout:directory}/srv
bin = ${buildout:directory}/bin
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = ${rootdirectory:var}/log
services = ${rootdirectory:etc}/run
run = ${rootdirectory:var}/run
[directory]
recipe = slapos.cookbook:mkdirectory
sphinx-data = ${rootdirectory:srv}/sphinx
stack/erp5/instance-tidstorage.cfg.in
View file @ 4b05a7da
......@@ -485,7 +485,6 @@ recipe = slapos.cookbook:publish
[tidstorage]
recipe = slapos.cookbook:tidstorage
known-tid-storage-identifier-dict = {{ known_tid_storage_identifier_dict }}
base-url = http://${zope-admin:ip}:${zope-admin:port}/%s/serialize
configuration-path = ${directory:etc}/tidstorage.py
ip = {{ ipv4 }}
port = 6001
......
stack/erp5/instance.cfg.in
View file @ 4b05a7da
......@@ -197,7 +197,6 @@ memcached = {{ template_memcached }}
cloudooo = ${dynamic-template-cloudooo:rendered}
zope = ${dynamic-template-zope:rendered}
mariadb = {{ template_mariadb }}
sphinx = {{ template_sphinx }}
tidstorage = ${dynamic-template-tidstorage:rendered}
varnish = ${dynamic-template-varnish:rendered}
......
stack/erp5/zope.conf.in
View file @ 4b05a7da
......@@ -43,10 +43,11 @@ products {{ instance_products }}
</product-config>
{% endif -%}
{% if 'timerserver-interval' in parameter_dict -%}
{% set timerserver_interval = parameter_dict.get('timerserver-interval', '0') | int -%}
{% if timerserver_interval -%}
%import timerserver
<timer-server>
interval {{ parameter_dict['timerserver-interval'] }}
interval {{ timerserver_interval }}
</timer-server>
{% endif -%}
......
stack/lamp/apache/instance-apache-import.cfg 0 → 100644
View file @ 4b05a7da
[buildout]
extends = ${template-apache-php:output}
${template-pbsready-export:output}
parts =
apache-proxy
logrotate
logrotate-entry-apache
cron
cron-entry-logrotate
sshkeys-authority
sshkeys-dropbear
dropbear-server
dropbear-server-pbs-authorized-key
[apache-proxy]
recipe = slapos.cookbook:apacheproxy
url = $${slap-parameter:proxy-url}
pid-file = $${basedirectory:run}/apache.pid
lock-file = $${basedirectory:run}/apache.lock
ip = $${slap-network-information:global-ipv6}
port = 8080
error-log = $${directory:httpd-log}/error.log
access-log = $${directory:httpd-log}/access.log
httpd-conf = $${rootdirectory:etc}/apache.conf
wrapper = $${basedirectory:services}/apache
promise = $${basedirectory:promises}/apache
httpd-binary = ${apache:location}/bin/httpd
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/
keys = $${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
wrapper = $${basedirectory:services}/sshkeys_authority
keygen-binary = ${dropbear:location}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = $${dropbear-server:wrapper}
public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
port = 2222
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${slap-parameter:authorized-key}
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:htdocs}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
[logrotate]
recipe = slapos.cookbook:logrotate
# Binaries
logrotate-binary = ${logrotate:location}/usr/sbin/logrotate
gzip-binary = ${gzip:location}/bin/gzip
gunzip-binary = ${gzip:location}/bin/gunzip
# Directories
wrapper = $${rootdirectory:bin}/logrotate
conf = $${rootdirectory:etc}/logrotate.conf
logrotate-entries = $${directory:logrotate-entries}
backup = $${directory:logrotate-backup}
state-file = $${rootdirectory:srv}/logrotate.status
[logrotate-entry-apache]
<= logrotate
recipe = slapos.cookbook:logrotate.d
name = apache
log = $${apache-proxy:error-log} $${apache-proxy:access-log}
frequency = daily
rotate-num = 30
sharedscripts = true
notifempty = true
create = true
[cron]
recipe = slapos.cookbook:cron
dcrond-binary = ${dcron:location}/sbin/crond
cron-entries = $${directory:cron-entries}
crontabs = $${directory:crontabs}
cronstamps = $${directory:cronstamps}
catcher = $${cron-simplelogger:wrapper}
binary = $${basedirectory:services}/crond
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
log = $${basedirectory:log}/crond.log
[cron-entry-logrotate]
<= cron
recipe = slapos.cookbook:cron.d
name = logrotate
frequency = 0 0 * * *
command = $${logrotate:wrapper}
[rootdirectory]
recipe = slapos.cookbook:mkdirectory
etc = $${buildout:directory}/etc/
var = $${buildout:directory}/var/
srv = $${buildout:directory}/srv/
bin = $${buildout:directory}/bin/
tmp = $${buildout:directory}/tmp/
[basedirectory]
recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
[directory]
recipe = slapos.cookbook:mkdirectory
htdocs = $${rootdirectory:srv}/www/
logrotate-entries = $${rootdirectory:etc}/logrotate.d/
logrotate-backup = $${basedirectory:backup}/logrotate/
cronstamps = $${rootdirectory:etc}/cronstamps/
cron-entries = $${rootdirectory:etc}/cron.d/
crontabs = $${rootdirectory:etc}/crontabs/
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
httpd-log = $${basedirectory:log}/apache/
stack/lamp/instance-apache-php.cfg → stack/lamp/apache/instance-apache-php.cfg
View file @ 4b05a7da
......@@ -107,7 +107,7 @@ software-url = $${slap-connection:software-release-url}
software-type = mariadb
return = url
sla = computer_guid
sla-computer_guid = ${slap-connection:computer-id}
sla-computer_guid = $${slap-connection:computer-id}
[mariadb-urlparse]
recipe = slapos.cookbook:urlparse
......
stack/lamp/buildout.cfg
View file @ 4b05a7da
[buildout]
ignore-existing = true
parts =
apache-php
mariadb
......@@ -9,13 +12,16 @@ parts =
eggs
instance-recipe-egg
template
template-apache-php
template-mariadb
template-mariadb-pbsready
template-mariadb-pbsready-import
template-mariadb-pbsready-export
#Contains the importer and exporter recipes for mariadb
template-mariadb-import
template-mariadb-export
extends =
../resilient/buildout.cfg
../../component/mariadb/buildout.cfg
../../component/apache/buildout.cfg
../../component/apache-php/buildout.cfg
......@@ -59,64 +65,60 @@ strip-top-level-dir = true
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg
md5sum = 8117f10e814a13c5376af4c01e6546d4
md5sum = 8b4660ccaccda1fa8b0e73b8ac38be11
mode = 0644
[template-apache-php]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-php.cfg
url = ${:_profile_base_location_}/apache/instance-apache-php.cfg
output = ${buildout:directory}/template-apache-php.cfg
md5sum = 59cee571ea0fcdf2e6c9b3195e258738
md5sum = a5dd222b3faa4e1ef2df9b3b9bb47966
mode = 0644
[template-apache-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-apache-backup.cfg
url = ${:_profile_base_location_}/apache/instance-apache-backup.cfg
output = ${buildout:directory}/template-apache-backup.cfg
md5sum = cfb77ac8785e0d125a785f69a5339014
mode = 0644
[template-backuped]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-backuped.cfg
output = ${buildout:directory}/template-backuped.cfg
md5sum = f43d1c6412ea8dc83b75573dc00daf9e
[template-resilient-lamp]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/instance-resilient.cfg
rendered = ${buildout:directory}/template-resilient.cfg
context = key templateapache template-apache-php:output
key dropbear dropbear:location
key buildout buildout:bin-directory
import-list = file parts template-parts:destination
file replicated template-replicated:destination
md5sum = 03aafcba5c626a4a1bd180d71007be1e
mode = 0644
[template-mariadb]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb.cfg
url = ${:_profile_base_location_}/mariadb/instance-mariadb.cfg
output = ${buildout:directory}/template-mariadb.cfg
md5sum = 767452bc503ff6c1b7af0ebfac590c9f
md5sum = fa9dc10efbcf61119f4cbab37c741322
mode = 0644
[template-mariadb-pbsready]
[template-mariadb-import]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pbsready.cfg
output = ${buildout:directory}/template-mariadb-pbsready.cfg
md5sum = d2a580dcd7efdd528be45c5ffadfe760
url = ${:_profile_base_location_}/mariadb/instance-mariadb-import.cfg
output = ${buildout:directory}/template-mariadb-import.cfg
md5sum = fa696733db4bd5b2e3e9fb6e0b09c59b
mode = 0644
[template-mariadb-pbsready-import]
[template-mariadb-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pbsready-import.cfg
output = ${buildout:directory}/template-mariadb-pbsready-import.cfg
md5sum = 4a96ff02da3898fef7077fa8baec81ac
url = ${:_profile_base_location_}/mariadb/instance-mariadb-export.cfg
output = ${buildout:directory}/template-mariadb-export.cfg
md5sum = 4b7dec765265b27c8235419b82ca7b02
mode = 0644
[template-mariadb-pbsready-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-mariadb-pbsready-export.cfg
output = ${buildout:directory}/template-mariadb-pbsready-export.cfg
md5sum = 11a9e45e8bc590bc11bfdd304b07a4a5
mode = 0644
[template-pull-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg
output = ${buildout:directory}/template-pull-backup.cfg
md5sum = 9aab30ba5aa23a37d4b507e7c414be00
mode = 0644
# Dummy parts in case no application configuration file is needed
[application-template]
......@@ -265,43 +267,45 @@ signature-certificate-list =
-----END CERTIFICATE-----
[versions]
# 1.2.4b doesn't download
MySQL-python = 1.2.3
Jinja2 = 2.6
Werkzeug = 0.8.3
apache-libcloud = 0.11.1
apache-libcloud = 0.11.3
async = 0.6.1
buildout-versions = 1.7
gitdb = 0.5.4
hexagonit.recipe.cmmi = 1.6
meld3 = 0.6.8
meld3 = 0.6.9
pycrypto = 2.6
rdiff-backup = 1.0.5
slapos.cookbook = 0.66
slapos.libnetworkcache = 0.13.2
slapos.recipe.build = 0.11.2
slapos.cookbook = 0.68
slapos.recipe.build = 0.11.5
slapos.recipe.download = 1.0.dev-r4053
slapos.recipe.template = 2.4.2
slapos.toolbox = 0.30
slapos.toolbox = 0.31
smmap = 0.8.2
# Required by:
# slapos.core==0.28.5
# slapos.toolbox==0.30
# slapos.core==0.31.2
# slapos.toolbox==0.31
Flask = 0.9
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31
GitPython = 0.3.2.RC1
# Required by:
# slapos.cookbook==0.66
# slapos.cookbook==0.68
PyXML = 0.8.4
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31
atomize = 0.1.1
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31
feedparser = 5.1.2
# Required by:
......@@ -309,58 +313,61 @@ feedparser = 5.1.2
hexagonit.recipe.download = 1.5.1
# Required by:
# slapos.cookbook==0.66
# slapos.cookbook==0.68
inotifyx = 0.2.0
# Required by:
# slapos.cookbook==0.66
# slapos.core==0.28.5
# slapos.cookbook==0.68
# slapos.core==0.31.2
# xml-marshaller==0.9.7
lxml = 3.0alpha2
# Required by:
# slapos.cookbook==0.66
# slapos.cookbook==0.68
netaddr = 0.7.10
# Required by:
# slapos.core==0.28.5
# slapos.core==0.31.2
netifaces = 0.8
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31
paramiko = 1.7.7.2
# Required by:
# slapos.toolbox==0.30
# slapos.toolbox==0.31
psutil = 0.6.1
# Required by:
# slapos.cookbook==0.66
# slapos.core==0.28.5
# slapos.toolbox==0.30
# zc.buildout==1.6.0-dev-SlapOS-006
# slapos.cookbook==0.68
pytz = 2012f
# Required by:
# slapos.cookbook==0.68
# slapos.core==0.31.2
# slapos.toolbox==0.31
# zc.buildout==1.6.0-dev-SlapOS-010
# zc.recipe.egg==1.3.2
setuptools = 0.6c12dev-r88846
# Required by:
# slapos.cookbook==0.66
# slapos.toolbox==0.30
slapos.core = 0.28.5
# slapos.cookbook==0.68
# slapos.toolbox==0.31
slapos.core = 0.31.2
# Required by:
# slapos.core==0.28.5
supervisor = 3.0a12
# slapos.core==0.31.2
supervisor = 3.0b1
# Required by:
# slapos.cookbook==0.66
# slapos.toolbox==0.30
# slapos.cookbook==0.68
# slapos.toolbox==0.31
xml-marshaller = 0.9.7
# Required by:
# slapos.cookbook==0.66
# slapos.cookbook==0.68
zc.recipe.egg = 1.3.2
# Required by:
# slapos.core==0.28.5
# slapos.core==0.31.2
zope.interface = 4.0.1
stack/lamp/instance-backuped.cfg deleted 100644 → 0
View file @ fb52bb07
[buildout]
extends =
${template-apache-php:output}
parts =
request-mariadb
request-mariadb-pseudo-replicating
request-apache-backup
request-pull-backup-server
request-pull-backup-server-mariadb
request-pull-backup-server-mariadb-backup
request-pull-backup-server-apache
request-pull-backup-server-apache-backup
url
apache-php
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-apache
logrotate-entry-stunnel
cron
cron-entry-logrotate
dropbear-server
sshkeys-authority
dropbear-server-pbs-authorized-key
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = $${directory:sshkeys}/requests/
keys = $${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = $${sshkeys-directory:requests}
keys-directory = $${sshkeys-directory:keys}
wrapper = $${basedirectory:services}/sshkeys_authority
keygen-binary = ${dropbear:location}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = $${dropbear-server:wrapper}
public-key = $${dropbear-server:rsa-keyfile}.pub
private-key = $${dropbear-server:rsa-keyfile}
wrapper = $${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = $${slap-network-information:global-ipv6}
port = 2222
home = $${directory:ssh}
wrapper = $${rootdirectory:bin}/raw_sshd
shell = $${rdiff-backup-server:wrapper}
rsa-keyfile = $${directory:ssh}/server_key.rsa
dropbear-binary = ${dropbear:location}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = $${request-pull-backup-server:connection-ssh-key}
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:htdocs}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
[request-pull-backup-server]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS (Pull Backup Server)
software-url = $${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key notification-url feeds-url
slave = false
[request-mariadb]
software-type = mariadb-pbsready-export
config = authorized-key notify
config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
config-notify = $${request-pull-backup-server:connection-notification-url}
return = url ssh-public-key ssh-url notification-id
[request-mariadb-pseudo-replicating]
<= slap-connection
recipe = slapos.cookbook:request
name = MariaDB Backup
software-url = $${slap-connection:software-release-url}
software-type = mariadb-pbsready-import
return = url ssh-public-key ssh-url notification-url
config = authorized-key on-notification
config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
pbs-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-push
config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${:pbs-notification-id}
[request-apache-backup]
<= slap-connection
recipe = slapos.cookbook:request
name = Apache Backup
software-url = $${slap-connection:software-release-url}
software-type = apache-backup
return = url ssh-url ssh-public-key
config = authorized-key proxy-url
config-authorized-key = $${request-pull-backup-server:connection-ssh-key}
config-proxy-url = $${url:url}
[request-pull-backup-server-apache]
<= request-pull-backup-server
slap-connection
recipe = slapos.cookbook:request
name = PBS pulling from Apache
software-url = $${slap-connection:software-release-url}
config = url name type server-key notify notification-id frequency
config-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache
config-type = pull
config-server-key = $${sshkeys-dropbear:public-key-value}
config-notify = $${request-pull-backup-server:connection-notification-url}
config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * *
slave = true
[request-pull-backup-server-apache-backup]
<= request-pull-backup-server
slap-connection
recipe = slapos.cookbook:request
name = PBS pushing to $${request-apache-backup:name}
software-url = $${slap-connection:software-release-url}
config = url name type server-key on-notification
config-url = $${request-apache-backup:connection-ssh-url}
config-name = $${request-pull-backup-server-apache:config-name}
config-type = push
config-server-key = $${request-apache-backup:connection-ssh-public-key}
config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-apache:config-notification-id}
slave = true
[request-pull-backup-server-mariadb]
<= request-pull-backup-server
slap-connection
name = PBS pulling from $${request-mariadb:name}
config = url name type server-key on-notification notify notification-id title
config-url = $${request-mariadb:connection-ssh-url}
config-name = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb
config-type = pull
config-server-key = $${request-mariadb:connection-ssh-public-key}
config-on-notification = $${request-mariadb:connection-notification-id}
config-notify = $${request-pull-backup-server:connection-notification-url}
config-notification-id = $${slap-connection:computer-id}-$${slap-connection:partition-id}-mariadb-pull
config-title = Pulling from MariaDB
slave = true
[request-pull-backup-server-mariadb-backup]
<= request-pull-backup-server
slap-connection
name = PBS pushing on $${request-mariadb-pseudo-replicating:name}
config = url name type server-key on-notification notify notification-id title
config-url = $${request-mariadb-pseudo-replicating:connection-ssh-url}
config-name = $${request-pull-backup-server-mariadb:config-name}
config-type = push
config-server-key = $${request-mariadb-pseudo-replicating:connection-ssh-public-key}
config-on-notification = $${request-pull-backup-server:connection-feeds-url}$${request-pull-backup-server-mariadb:config-notification-id}
config-notify = $${request-mariadb-pseudo-replicating:connection-notification-url}
config-notification-id = $${request-mariadb-pseudo-replicating:pbs-notification-id}
config-title = Pushing to MariaDB backup
slave = true
[directory]
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
stack/lamp/instance-mariadb-pbsready-export.cfg deleted 100644 → 0
View file @ fb52bb07
[buildout]
extends = ${template-mariadb-pbsready:output}
parts += cron-entry-mariadb-backup
[urls]
notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-mydumper:name}
[mydumper]
recipe = slapos.cookbook:mydumper
wrapper = $${rootdirectory:bin}/raw_mydumper
backup-directory = $${directory:mariadb-backup}
socket = $${mariadb:socket}
user = root
mydumper-binary = ${mydumper:location}/bin/mydumper
database = $${mariadb:database}
import = false
[notifier-mydumper]
<= notifier
recipe = slapos.cookbook:notifier.notify
name = mydumper
title = Dumping MariaDB Database
executable = $${mydumper:wrapper}
wrapper = $${rootdirectory:bin}/mydumper
notify = $${slap-parameter:notify}
[cron-entry-mariadb-backup]
<= cron
recipe = slapos.cookbook:cron.d
name = backup
frequency = 0 * * * *
command = $${notifier-mydumper:wrapper}
stack/lamp/instance-resilient.cfg 0 → 100644
View file @ 4b05a7da
{% import 'parts' as parts %}
{% import 'replicated' as replicated %}
[buildout]
extends =
{{templateapache}}
parts =
{{ parts.replicate("mariadb","3") }}
request-apache-backup-1
request-apache-backup-2
request-pull-backup-server-apache-1
request-pull-backup-server-apache-backup-1
request-pull-backup-server-apache-2
request-pull-backup-server-apache-backup-2
publish-connection-informations
apache-php
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-apache
logrotate-entry-stunnel
cron
cron-entry-logrotate
dropbear-server
sshkeys-authority
dropbear-server-pbs-authorized-key
request-pull-backup-server
{{ replicated.replicate("mariadb", "3", "mariadb-export", "mariadb-import") }}
[request-pull-backup-server]
<= request-pbs-common
name = PBS (Pull Backup Server)
return = ssh-key notification-url feeds-url
slave = false
[sshkeys-directory]
recipe = slapos.cookbook:mkdirectory
requests = ${directory:sshkeys}/requests/
keys = ${directory:sshkeys}/keys/
[sshkeys-authority]
recipe = slapos.cookbook:sshkeys_authority
request-directory = ${sshkeys-directory:requests}
keys-directory = ${sshkeys-directory:keys}
wrapper = ${basedirectory:services}/sshkeys_authority
keygen-binary = {{dropbear}}/bin/dropbearkey
[sshkeys-dropbear]
<= sshkeys-authority
recipe = slapos.cookbook:sshkeys_authority.request
name = dropbear
type = rsa
executable = ${dropbear-server:wrapper}
public-key = ${dropbear-server:rsa-keyfile}.pub
private-key = ${dropbear-server:rsa-keyfile}
wrapper = ${basedirectory:services}/sshd
[dropbear-server]
recipe = slapos.cookbook:dropbear
host = ${slap-network-information:global-ipv6}
port = 2222
home = ${directory:ssh}
wrapper = ${rootdirectory:bin}/raw_sshd
shell = ${rdiff-backup-server:wrapper}
rsa-keyfile = ${directory:ssh}/server_key.rsa
dropbear-binary = {{dropbear}}/sbin/dropbear
[dropbear-server-pbs-authorized-key]
<= dropbear-server
recipe = slapos.cookbook:dropbear.add_authorized_key
key = ${request-pull-backup-server:connection-ssh-key}
[rdiff-backup-server]
<= apache-php
recipe = slapos.cookbook:pbs
client = false
path = ${apache-php:htdocs}
wrapper = ${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = {{buildout}}/rdiff-backup
[request-apache-backup-1]
<= slap-connection
recipe = slapos.cookbook:request
name = Apache Backup 1
software-url = ${slap-connection:software-release-url}
software-type = apache-backup
return = url ssh-url ssh-public-key
config = authorized-key proxy-url
config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
config-proxy-url = ${publish-connection-informations:url}
[request-apache-backup-2]
<= slap-connection
recipe = slapos.cookbook:request
name = Apache Backup 2
software-url = ${slap-connection:software-release-url}
software-type = apache-backup
return = url ssh-url ssh-public-key
config = authorized-key proxy-url
config-authorized-key = ${request-pull-backup-server:connection-ssh-key}
config-proxy-url = ${publish-connection-informations:url}
[request-pull-backup-server-apache-1]
<= request-pbs-common
name = PBS pulling from Apache 1
config = url name type server-key notify notification-id frequency
config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
config-type = pull
config-server-key = ${sshkeys-dropbear:public-key-value}
config-notify = ${request-pull-backup-server:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * *
slave = true
[request-pull-backup-server-apache-2]
<= request-pbs-common
name = PBS pulling from Apache 2
config = url name type server-key notify notification-id frequency
config-url = ssh://nobody@[${dropbear-server:host}]:${dropbear-server:port}/${rdiff-backup-server:path}
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache
config-type = pull
config-server-key = ${sshkeys-dropbear:public-key-value}
config-notify = ${request-pull-backup-server:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-apache-pull
config-frequency = 30 * * * *
slave = true
[request-pull-backup-server-apache-backup-1]
<= request-pbs-common
name = PBS pushing to ${request-apache-backup-1:name}
config = url name type server-key on-notification
config-url = ${request-apache-backup-1:connection-ssh-url}
config-name = ${request-pull-backup-server-apache-1:config-name}
config-type = push
config-server-key = ${request-apache-backup-1:connection-ssh-public-key}
config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-1:config-notification-id}
slave = true
[request-pull-backup-server-apache-backup-2]
<= request-pbs-common
name = PBS pushing to ${request-apache-backup-2:name}
config = url name type server-key on-notification
config-url = ${request-apache-backup-2:connection-ssh-url}
config-name = ${request-pull-backup-server-apache-2:config-name}
config-type = push
config-server-key = ${request-apache-backup-2:connection-ssh-public-key}
config-on-notification = ${request-pull-backup-server:connection-feeds-url}${request-pull-backup-server-apache-2:config-notification-id}
slave = true
[directory]
ssh = ${rootdirectory:etc}/ssh/
sshkeys = ${rootdirectory:srv}/sshkeys
stack/lamp/instance.cfg
View file @ 4b05a7da
[buildout]
extends =
${template-switchsoftware:output}
parts =
switch_softwaretype
......@@ -9,9 +13,11 @@ offline = true
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
default = ${template-apache-php:output}
backuped = ${template-backuped:output}
resilient = ${template-resilient-lamp:rendered}
mariadb = ${template-mariadb:output}
mariadb-pbsready-import = ${template-mariadb-pbsready-import:output}
mariadb-pbsready-export = ${template-mariadb-pbsready-export:output}
mariadb-import = ${template-mariadb-import:output}
mariadb-export = ${template-mariadb-export:output}
pull-backup = ${template-pull-backup:output}
apache-backup = ${template-apache-backup:output}
frozen = ${template-frozen:output}
stack/lamp/mariadb/instance-mariadb-export.cfg 0 → 100644
View file @ 4b05a7da
[buildout]
extends = ${template-mariadb:output}
${template-pbsready-export:output}
parts += mariadb
[exporter]
recipe = slapos.cookbook:mydumper
wrapper = $${rootdirectory:bin}/$${slap-parameter:namebase}-exporter
backup-directory = $${directory:backup}
socket = $${mariadb:socket}
user = root
mydumper-binary = ${mydumper:location}/bin/mydumper
database = $${mariadb:database}
import = false
\ No newline at end of file
stack/lamp/instance-mariadb-pbsready-import.cfg → stack/lamp/mariadb/instance-mariadb-import.cfg
View file @ 4b05a7da
[buildout]
extends = ${template-mariadb:output}
${template-pbsready-import:output}
extends = ${template-mariadb-pbsready:output}
parts += mariadb
parts += mariadb-import-on-notification
[urls]
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
[mydumper-import]
[importer]
recipe = slapos.cookbook:mydumper
wrapper = $${rootdirectory:bin}/myloader
backup-directory = $${directory:mariadb-backup}
backup-directory = $${directory:backup}
socket = $${mariadb:socket}
user = root
myloader-binary = ${mydumper:location}/bin/myloader
database = $${mariadb:database}
import = true
[mariadb-import-on-notification]
<= notifier
recipe = slapos.cookbook:notifier.callback
on-notification-id = $${slap-parameter:on-notification}
callback = $${mydumper-import:wrapper}
stack/lamp/instance-mariadb.cfg → stack/lamp/mariadb/instance-mariadb.cfg
View file @ 4b05a7da
......@@ -23,6 +23,7 @@ offline = true
[urls]
recipe = slapos.cookbook:publish
url = mysqls://$${mariadb:user}:$${mariadb:password}@[$${stunnel:remote-host}]:$${stunnel:remote-port}/$${mariadb:database}
ip = $${slap-network-information:global-ipv6}
[mariadb]
recipe = slapos.cookbook:mysql
......@@ -52,6 +53,7 @@ mysql-install-binary = ${mariadb:location}/scripts/mysql_install_db
mysql-upgrade-binary = ${mariadb:location}/bin/mysql_upgrade
mysqld-binary = ${mariadb:location}/bin/mysqld
[slapmonitor]
recipe = slapos.cookbook:slapmonitor
pid-file = $${basedirectory:run}/mariadb.pid
......@@ -195,6 +197,7 @@ recipe = slapos.cookbook:mkdirectory
log = $${rootdirectory:var}/log/
services = $${rootdirectory:etc}/run/
run = $${rootdirectory:var}/run/
script = $${rootdirectory:etc}/script/
backup = $${rootdirectory:srv}/backup/
promises = $${rootdirectory:etc}/promise/
......
stack/resilient/buildout.cfg 0 → 100644
View file @ 4b05a7da
[buildout]
parts =
#Templates needed to setup automatic backup
template-pbsready
template-pbsready-import
template-pbsready-export
template-replicated
template-parts
#Frozen is the state used to not destroy a broken instance's content
template-frozen
template-resilient
template-switchsoftware
[template-pbsready]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready.cfg
output = ${buildout:directory}/template-pbsready.cfg
md5sum = 45e64cfb6afbcfda1f9f85e33c73bd99
mode = 0644
[template-pbsready-import]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready-import.cfg
output = ${buildout:directory}/template-pbsready-import.cfg
md5sum = 5ba7477f9499a7dbde5f33ca96bd6ba4
mode = 0644
[template-pbsready-export]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pbsready-export.cfg
output = ${buildout:directory}/template-pbsready-export.cfg
md5sum = 29d36aac2008b173cb9ce5da9e88c0fa
mode = 0644
[template-pull-backup]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg
output = ${buildout:directory}/template-pull-backup.cfg
md5sum = f88cc9192a63c88f83a9e5191075534e
mode = 0644
[template-replicated]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-replicated.cfg
md5sum = 61780842ccda1600a102456b0da69ac9
mode = 0644
destination = ${buildout:directory}/template-replicated.cfg
[template-parts]
recipe = slapos.recipe.download
url = ${:_profile_base_location_}/template-parts.cfg
md5sum = f5fc27235725f05fdbde76a78ebc363e
mode = 0644
destination = ${buildout:directory}/template-parts.cfg
[template-frozen]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-frozen.cfg
output = ${buildout:directory}/template-frozen.cfg
[template-resilient]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/resilient.cfg
output = ${buildout:directory}/resilient.cfg
md5sum = 59e74d290d623de2c1e147e48f284fba
mode = 0644
[template-switchsoftware]
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/switchsoftware.cfg
output = ${buildout:directory}/switchsoftware.cfg
md5sum = c94a0ed85fce2e72254ae956dce7e40d
mode = 0644
\ No newline at end of file
stack/resilient/instance-frozen.cfg 0 → 100644
View file @ 4b05a7da
[buildout]
parts =
\ No newline at end of file
stack/resilient/instance-pbsready-export.cfg 0 → 100644
View file @ 4b05a7da
[buildout]
extends = ${template-pbsready:output}
parts += cron-entry-backup
[urls]
notification-id = http://[$${notifier:host}]:$${notifier:port}/get/$${notifier-exporter:name}
#notify launches executable, and once it's done, notifies the pull-backup-servers.
[notifier-exporter]
<= notifier
recipe = slapos.cookbook:notifier.notify
name = exporter
title = Dumping $${slap-parameter:namebase}
executable = $${exporter:wrapper}
wrapper = $${rootdirectory:bin}/exporter
notify = $${slap-parameter:notify}
#adds the exporter to cron
[cron-entry-backup]
<= cron
recipe = slapos.cookbook:cron.d
name = backup
frequency = 0 * * * *
command = $${notifier-exporter:wrapper}
stack/resilient/instance-pbsready-import.cfg 0 → 100644
View file @ 4b05a7da
[buildout]
extends = ${template-pbsready:output}
parts += import-on-notification
[urls]
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
#Launches callback, when a notification is received
[import-on-notification]
<= notifier
recipe = slapos.cookbook:notifier.callback
on-notification-id = $${slap-parameter:on-notification}
callback = $${importer:wrapper}
stack/lamp/instance-mariadb-pbsready.cfg → stack/resilient/instance-pbsready.cfg
View file @ 4b05a7da
[buildout]
extends = ${template-mariadb:output}
parts =
resiliency
urls
mariadb
stunnel
certificate-authority
ca-stunnel
logrotate
logrotate-entry-mariadb
logrotate-entry-stunnel
logrotate-entry-cron
logrotate-entry-equeue
......@@ -21,6 +18,13 @@ parts =
dropbear-server-pbs-authorized-key
notifier
# adds the resiliency script for the bully algorithm
[resiliency]
recipe = slapos.cookbook:addresiliency
script = $${basedirectory:script}
run = $${basedirectory:services}
# sets up an rdiff-backup server (with a dropbear server for ssh)
[urls]
ssh-public-key = $${sshkeys-dropbear:public-key-value}
ssh-url = ssh://nobody@[$${dropbear-server:host}]:$${dropbear-server:port}/$${rdiff-backup-server:path}
......@@ -65,10 +69,12 @@ key = $${slap-parameter:authorized-key}
[rdiff-backup-server]
recipe = slapos.cookbook:pbs
client = false
path = $${directory:mariadb-backup}
path = $${directory:backup}
wrapper = $${rootdirectory:bin}/rdiffbackup-server
rdiffbackup-binary = ${buildout:bin-directory}/rdiff-backup
## Sets up the execution queue for the notifier
[logrotate-entry-equeue]
<= logrotate
recipe = slapos.cookbook:logrotate.d
......@@ -85,6 +91,9 @@ database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue
## notifier.notify adds the [exporter, notifier] to the execution queue
## notifier.notify.callback sets up a callback
[notifier]
recipe = slapos.cookbook:notifier
feeds = $${directory:notifier-feeds}
......@@ -99,12 +108,15 @@ notifier-binary = ${buildout:bin-directory}/pubsubnotifier
[basedirectory]
script = $${rootdirectory:etc}/script/
services = $${rootdirectory:etc}/run/
cache = $${rootdirectory:var}/cache/
notifier = $${rootdirectory:etc}/notifier/
[directory]
mariadb-backup = $${basedirectory:backup}/mariadb/
backup = $${basedirectory:backup}/$${slap-parameter:namebase}
ssh = $${rootdirectory:etc}/ssh/
sshkeys = $${rootdirectory:srv}/sshkeys
notifier-feeds = $${basedirectory:notifier}/feeds/
notifier-callbacks = $${basedirectory:notifier}/callbacks/
script = $${basedirectory:script}
stack/lamp/instance-pull-backup.cfg → stack/resilient/instance-pull-backup.cfg
View file @ 4b05a7da
......@@ -19,6 +19,8 @@ ssh-key = $${sshkeys-dropbear:public-key-value}
notification-url = http://[$${notifier:host}]:$${notifier:port}/notify
feeds-url = http://[$${notifier:host}]:$${notifier:port}/get/
## sets up the equeue for the notifier
[equeue]
recipe = slapos.cookbook:equeue
socket = $${basedirectory:run}/equeue.sock
......@@ -27,6 +29,7 @@ database = $${rootdirectory:srv}/equeue.db
wrapper = $${basedirectory:services}/equeue
equeue-binary = ${buildout:bin-directory}/equeue
[notifier]
recipe = slapos.cookbook:notifier
feeds = $${directory:notifier-feeds}
......@@ -39,6 +42,8 @@ wrapper = $${basedirectory:services}/notifier
server-binary = ${buildout:bin-directory}/pubsubserver
notifier-binary = ${buildout:bin-directory}/pubsubnotifier
## Dropbear Client to provide ssh
[dropbear-client]
recipe = slapos.cookbook:dropbear.client
dbclient-binary = ${dropbear:location}/bin/dbclient
......@@ -68,6 +73,9 @@ public-key = $${dropbear-client:identity-file}.pub
private-key = $${dropbear-client:identity-file}
wrapper = $${rootdirectory:bin}/do_backup
## The pull-backup-server contains every backup (incremental).
## to prevent a corrupt dump from destroying everything.
[pbs]
<= notifier
recipe = slapos.cookbook:pbs
......@@ -82,6 +90,7 @@ wrappers-directory = $${directory:pbs-wrappers}
notifier-url = http://[$${notifier:host}]:$${notifier:port}/
slave-instance-list = $${slap-parameter:slave_instance_list}
[cron-simplelogger]
recipe = slapos.cookbook:simplelogger
wrapper = $${rootdirectory:bin}/cron_simplelogger
......@@ -170,3 +179,8 @@ notifier = $${rootdirectory:etc}/notifier/
[rootdirectory]
home = $${buildout:directory}/home/
# Default values
[slap-parameter]
slave_instance_list = []
stack/resilient/resilient.cfg 0 → 100644
View file @ 4b05a7da
## not used at the moment
[buildout]
parts =
request-pull-backup-server
[request-pull-backup-server]
<= slap-connection
recipe = slapos.cookbook:request
name = PBS (Pull Backup Server)
software-url = $${slap-connection:software-release-url}
software-type = pull-backup
return = ssh-key notification-url feeds-url
slave = false
\ No newline at end of file
stack/resilient/switchsoftware.cfg 0 → 100644
View file @ 4b05a7da
[buildout]
parts =
switch_softwaretype
[switch_softwaretype]
recipe = slapos.cookbook:softwaretype
pull-backup = ${template-pull-backup:output}
#frozen creates a syntax error, meaning it can keep its data.
#It's dirty as hell, it needs to be replaced.
frozen = ${template-frozen:output}
stack/resilient/template-parts.cfg 0 → 100644
View file @ 4b05a7da
## Parts Needed for the resiliency
{% macro replicate(namebase, nbbackup) %}
request-{{namebase}}
request-{{namebase}}-2
{% for i in range(1,nbbackup|int) %}
request-{{namebase}}-pseudo-replicating-{{i}}
request-{{namebase}}-pseudo-replicating-{{i}}-2
{% endfor %}
{% for i in range(1,nbbackup|int) %}
request-pbs-{{namebase}}-{{i}}
request-pull-backup-server-{{namebase}}-{{i}}
request-pull-backup-server-{{namebase}}-backup-{{i}}
{% endfor %}
{% endmacro %}
\ No newline at end of file
stack/resilient/template-replicated.cfg 0 → 100644
View file @ 4b05a7da
{% macro replicate(namebase, nbbackup, typeexport, typeimport, heriteLeader='', heriteBackup='') %}
## Tells the Backupable recipe that we want a backup
[resilient]
config-script = bully.py
config-wrapper = bully
config-namebase = {{namebase}}
## Every request is double to provide the 3 IPs.
[request-{{namebase}}]
<= resilient
slap-connection
{{heriteLeader}}
software-type = {{typeexport}}
name = {{namebase}}0
return = url ssh-public-key ssh-url notification-id ip
config = number script wrapper authorized-key notify ip-list namebase
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-ip-list =
config-number = 0
{% for id in range(1,nbbackup|int) %}
[request-{{namebase}}-pseudo-replicating-{{id}}]
<= slap-connection
resilient
{{heriteBackup}}
recipe = slapos.cookbook:request
name = {{namebase}}{{id}}
software-url = ${slap-connection:software-release-url}
software-type = {{typeimport}}
return = url ssh-public-key ssh-url notification-url ip
config = number script wrapper authorized-key on-notification ip-list namebase
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
config-ip-list =
config-number = {{id}}
{% endfor %}
[iplist]
config-ip-list = ${request-{{namebase}}:connection-ip}{% for j in range(1,nbbackup|int) %} ${request-{{namebase}}-pseudo-replicating-{{j}}:connection-ip}{% endfor %}
[request-{{namebase}}-2]
<= resilient
slap-connection
iplist
{{heriteLeader}}
recipe = slapos.cookbook:request
name = {{namebase}}0
config = number script wrapper authorized-key notify ip-list namebase
software-url = ${slap-connection:software-release-url}
software-type = {{typeexport}}
return = url ssh-public-key ssh-url notification-id ip
config-authorized-key = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}{% endfor %}
config-notify = {% for id in range(1,nbbackup|int) %} ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}{% endfor %}
config-number=0
{% for id in range(1,nbbackup|int) %}
[request-{{namebase}}-pseudo-replicating-{{id}}-2]
<= slap-connection
resilient
iplist
{{heriteBackup}}
recipe = slapos.cookbook:request
name = {{namebase}}{{id}}
software-url = ${slap-connection:software-release-url}
software-type = {{typeimport}}
return = url ssh-public-key ssh-url notification-url
config = number script wrapper authorized-key on-notification ip-list namebase
config-authorized-key = ${request-pbs-{{namebase}}-{{id}}:connection-ssh-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${:pbs-notification-id}
pbs-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-push
config-number = {{id}}
{% endfor %}
## The PBS and their push / pull slaves
## Adding a PBS provides resiliency
## Adding a backup server provides availability
## Having 3 backups pulling from the same PBS provides
##only availability, not resiliency
## WARNING : SLAVES ARE ALLOCATED AT RANDOM, THIS NEEDS TO BE FIXED.
[request-pbs-common]
<= slap-connection
recipe = slapos.cookbook:request
software-url = ${slap-connection:software-release-url}
software-type = pull-backup
{% for id in range(1,nbbackup|int) %}
[request-pbs-{{namebase}}-{{id}}]
<= request-pbs-common
name = PBS ({{namebase}} / {{id}})
return = ssh-key notification-url feeds-url
slave = false
[request-pull-backup-server-{{namebase}}-{{id}}]
<= request-pbs-common
name = PBS {{id}} pulling from ${request-{{namebase}}:name}
config = url name type server-key on-notification notify notification-id title
config-url = ${request-{{namebase}}:connection-ssh-url}
config-name = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}
config-type = pull
config-server-key = ${request-{{namebase}}:connection-ssh-public-key}
config-on-notification = ${request-{{namebase}}:connection-notification-id}
config-notify = ${request-pbs-{{namebase}}-{{id}}:connection-notification-url}
config-notification-id = ${slap-connection:computer-id}-${slap-connection:partition-id}-{{namebase}}-{{id}}-pull
config-title = Pulling from {{namebase}}
slave = true
[request-pull-backup-server-{{namebase}}-backup-{{id}}]
<= request-pbs-common
name = PBS pushing on ${request-{{namebase}}-pseudo-replicating-{{id}}:name}
config = url name type server-key on-notification notify notification-id title
config-url = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-url}
config-name = ${request-pull-backup-server-{{namebase}}-{{id}}:config-name}
config-type = push
config-server-key = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-ssh-public-key}
config-on-notification = ${request-pbs-{{namebase}}-{{id}}:connection-feeds-url}${request-pull-backup-server-{{namebase}}-{{id}}:config-notification-id}
config-notify = ${request-{{namebase}}-pseudo-replicating-{{id}}:connection-notification-url}
config-notification-id = ${request-{{namebase}}-pseudo-replicating-{{id}}:pbs-notification-id}
config-title = Pushing to {{namebase}} backup {{id}}
slave = true
{% endfor %}
{% endmacro %}
\ No newline at end of file
stack/slapos.cfg
View file @ 4b05a7da
......@@ -24,6 +24,7 @@ find-links +=
# Use only quite well working sites.
allow-hosts +=
*.googlecode.com
*.nexedi.org
*.python.org
*.sourceforge.net
......@@ -34,7 +35,6 @@ allow-hosts +=
github.com
launchpad.net
peak.telecommunity.com
psutil.googlecode.com
sourceforge.net
www.dabeaz.com
www.owlfish.com
......@@ -59,7 +59,7 @@ eggs =
[versions]
# Use SlapOS patched zc.buildout
zc.buildout = 1.6.0-dev-SlapOS-007
zc.buildout = 1.6.0-dev-SlapOS-010
[networkcache]
download-cache-url = http://www.shacache.org/shacache
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7