Skip to content

slapos
slapos
Commit eb1f843f authored by Łukasz Nowak's avatar Łukasz Nowak
Browse files
Options

Update Release Candidate

parents 07bbdab1 de1e9ea0
Show whitespace changes
Inline Side-by-side
Showing with 236 additions and 1548 deletions
component/condor/buildout.cfg
View file @ eb1f843f
......@@ -11,7 +11,6 @@ extends =
../openldap/buildout.cfg
../dash/buildout.cfg
../zlib/buildout.cfg
../../stack/slapos.cfg
[condor]
recipe = slapos.recipe.build
......
component/gateone/buildout.cfg
View file @ eb1f843f
[buildout]
extends =
../../stack/slapos.cfg
../lxml-python/buildout.cfg
../python-kerberos/buildout.cfg
../git/buildout.cfg
......
component/gcc/buildout.cfg
View file @ eb1f843f
......@@ -8,7 +8,6 @@ extends =
../perl/buildout.cfg
../tar/buildout.cfg
../binutils/buildout.cfg
../zlib/buildout.cfg
parts =
gcc-8.2
......@@ -21,6 +20,8 @@ pre-configure =
set %(location)s/bin
mkdir -p $1
ln -s ${binutils:location}/bin/ld $1/ld
# remove "dependency_libs=' -lz'" line (-lz not needed anymore because the internal zlib was used)
sed -i '/^libbacktrace\.la:/{ N; s,$,\n\tsed -i /^dependency_libs=/d $@,; }' libbacktrace/Makefile.in
configure-options =
--disable-bootstrap
--disable-multilib
......@@ -31,6 +32,7 @@ configure-options =
--with-isl=${isl:location}
--with-ld=@@LOCATION@@/bin/ld
--with-as=${binutils:location}/bin/as
--with-internal-zlib
post-install =
cd '%(location)s/bin'
ln -s gcc cc
......@@ -51,8 +53,7 @@ post-install =
EOF
chmod +x ld
environment =
CPPFLAGS=-I${zlib:location}/include
LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
LDFLAGS=-Wl,-rpath=${gmp:location}/lib -Wl,-rpath=${isl:location}/lib -Wl,-rpath=${mpc:location}/lib -Wl,-rpath=${mpfr:location}/lib
PATH=${binutils:location}/bin:${gettext:location}/bin:${perl:location}/bin:${tar:location}/bin:%(PATH)s
[gcc-5.5]
......
component/mariadb/buildout.cfg
View file @ eb1f843f
......@@ -100,8 +100,8 @@ pre-configure =
cd fake_mariadb_source
ln -s ${mariadb:location}/wsrep-lib
cp -a ${mariadb:location}/include/mysql/server include
cp -a include/private sql
chmod -R a+w include sql # so that buildout can delete this compile-dir after install
chmod -R u+w include # so that buildout can delete this compile-dir after install
mv include/private sql
mkdir -p ${:plugin-dir}
configure-options =
--with-mysql-source=fake_mariadb_source
......@@ -109,7 +109,7 @@ configure-options =
--disable-static
--disable-document
post-install =
cp -ra ${mariadb:location}/lib/plugin/* ${:plugin-dir}
cp -rs ${mariadb:location}/lib/plugin/* ${:plugin-dir}
plugin-dir = @@LOCATION@@/lib/plugin
install-sql = @@LOCATION@@/share/mroonga/install.sql
make-targets = plugindir=${:plugin-dir} install
......
component/python3/buildout.cfg
View file @ eb1f843f
......@@ -20,7 +20,7 @@ parts =
python3
[python3]
<= python3.6
<= python3.7
[python3-common]
recipe = slapos.recipe.cmmi
......@@ -61,12 +61,7 @@ md5sum = 986078f11b39074be22a199e56491d98
[python3.7]
<= python3-common
version = 3.7
package_version = 3.7.2
md5sum = df6ec36011808205beda239c72f947cb
package_version = 3.7.7
md5sum = 172c650156f7bea68ce31b2fd01fa766
patch-options =
patches =
environment =
PATH=${xz-utils:location}/bin:%(PATH)s
CPPFLAGS=-I${zlib:location}/include -I${xz-utils:location}/include -I${readline:location}/include -I${libexpat:location}/include -I${libffi:location}/include -I${ncurses:location}/include -I${ncurses:location}/include/ncursesw -I${bzip2:location}/include -I${gdbm:location}/include -I${openssl:location}/include -I${sqlite3:location}/include -I${gettext:location}/include
LDFLAGS=-L${zlib:location}/lib -L${xz-utils:location}/lib -L${readline:location}/lib -L${libexpat:location}/lib -L${libffi:location}/lib -L${ncurses:location}/lib -L${bzip2:location}/lib -L${gdbm:location}/lib -L${openssl:location}/lib -L${sqlite3:location}/lib -Wl,-rpath=${zlib:location}/lib -Wl,-rpath=${xz-utils:location}/lib -Wl,-rpath=${readline:location}/lib -Wl,-rpath=${libexpat:location}/lib -Wl,-rpath=${libffi:location}/lib -Wl,-rpath=${ncurses:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${gdbm:location}/lib -Wl,-rpath=${openssl:location}/lib -Wl,-rpath=${sqlite3:location}/lib -L${gettext:location}/lib -Wl,-rpath=${gettext:location}/lib -Wl,-rpath=${file:location}/lib
# LD_LIBRARY_PATH=${libpng:location}/lib
component/r-language/software.cfg
View file @ eb1f843f
......@@ -3,8 +3,7 @@
[buildout]
extends =
../../stack/slapos.cfg
../../component/gcc/buildout.cfg
../../component/r-language/buildout.cfg
buildout.cfg
parts =
eggs
......
component/boinc-client/buildout.cfg → component/unstable/boinc-client/buildout.cfg
View file @ eb1f843f
......@@ -14,7 +14,6 @@ extends =
../pkgconfig/buildout.cfg
../openldap/buildout.cfg
../gnutls/buildout.cfg
../../stack/slapos.cfg
[download-boinc]
recipe = slapos.recipe.build:download
......
component/boinc/buildout.cfg → component/unstable/boinc/buildout.cfg
View file @ eb1f843f
......@@ -16,11 +16,6 @@ extends =
../gnutls/buildout.cfg
../mariadb/buildout.cfg
../curl/buildout.cfg
../../stack/slapos.cfg
[mariadb]
# Compile dir is for plugins, not needed in boinc
keep-compile-dir = false
#checkout boinc package from svn repository
[boinc-package]
......
setup.py
View file @ eb1f843f
......@@ -81,11 +81,6 @@ setup(name=name,
'apachephp = slapos.recipe.apachephp:Recipe',
'apachephpconfigure = slapos.recipe.apachephpconfigure:Recipe',
'apacheproxy = slapos.recipe.apacheproxy:Recipe',
'boinc = slapos.recipe.boinc:Recipe',
'boinc.app = slapos.recipe.boinc:App',
'boinc.client = slapos.recipe.boinc:Client',
'bonjourgrid = slapos.recipe.bonjourgrid:Recipe',
'bonjourgrid.client = slapos.recipe.bonjourgrid:Client',
'certificate_authority = slapos.recipe.certificate_authority:Recipe',
'certificate_authority.request = slapos.recipe.certificate_authority:Request',
'check_page_content = slapos.recipe.check_page_content:Recipe',
......
slapos/recipe/README.lamp.rst deleted 100644 → 0
View file @ 07bbdab1
lamp
=====
The lamp recipe helps you to deploy simply a php based application on slapos. This recipe is
able to setup mariadb, apache and apache-php for your php application, and is also capable of
configuring your software during installation to ensure a full compatibility.
How to use lamp ?
-----------------
just add this part in your software.cfg to use the lamp.simple module
.. code-block:: ini
[instance-recipe]
egg = slapos.cookbook
module = lamp.simple
you also need to extend lamp.cfg
.. code-block:: ini
extends =
https://lab.nexedi.com/nexedi/slapos/raw/slapos-0.50/stack/lamp.cfg
lamp.runner
===========
When you install some software (such as prestashop) you need to remove or rename folder, with slapos you can not
access to the www-data directory. to do this, you need to tell to lamp recipe to remove or/and it when software
will be instantiated. Some software requires more than rename or delete a folder (manualy create database etc...)
in this case you need to write a python script and lamp recipe must run it when installing your software.
How to use lamp.runner ?
------------------------
this part of lamp recipe work with slapos.toolbox, Therefore you must add it to your recipe.
in software.cfg, replace instance-recipe-egg part by
.. code-block:: ini
[instance-recipe-egg]
recipe = zc.recipe.egg
eggs =
${python-mysqlclient:egg}
${instance-recipe:egg}
slapos.toolbox[lampconfigure]
and add into your instance.cfg
.. code-block:: ini
lampconfigure_directory = ${buildout:bin-directory}/lampconfigure
CONDITION
----------
the action (delete, rename, script, chmod) only starts when the condition is filled.
in instance.cfg, add::
file_token = path_of_file
and the action will begin when path_of_www-data/path_of_file will be created
you can also use database to check condition. add ::
table_name = name_of_table
constraint = sql_where_condition
``name_of_table`` is the full or partial name(in some cases we can not know the prefix used to create tables) of table
into mariadb databse for example ``table_name = admin``. if you use
``name_of_table = **``, the action will begin when database is ready.
constraint is the sql_condition to use when search entry into name_of_table for example constraint = ``admin_id=1``
you can't use file_token and table_name at the same time, otherwise file_token will be used in priority. Beware of conditions that will never be satisfied.
ACTION
-------
The action starts when condition is true
1- delete file or folder into instance.cfg, use::
delete = file_or_folder1, file_or_folder2, file_or_folder3 ...
for example delete = admin
2- rename file or folder
into instance.cfg, use
rename_chmod = mode (optional)
rename = old_name1 => new_name1, old_name2 => new_name2, ...
you can also use
rename = old_name1, old_name2 => new_name2, ... in this case old_name1 will be rename and the new name will be chose
by joining old_name1 and mysql_user: this should give
rename = old_name1 => old_name1-mysql_user, old_name2 => new_name2, ...
use rename_chmod to apply a mode to new_name1, new_name2, ...
3- Change mode of file or directory
this is usefull if you want to change the mode of files into your www-data, use
chmod = file1, file2, ....
mode = mode_to_apply (ex= 0644)
4- Launch python script
use script = ${configure-script:location}/${configure-script:filename} into instance.cfg, add part configure-script
into software.cfg
.. code-block:: ini
parts = configure-script
[configure-script]
recipe = hexagonit.recipe.download
location = ${buildout:parts-directory}/${:_buildout_section_name_}
url = url_of_script_name.py
filename = script_name.py
download-only = True
The script_name.py should contain a main function, sys.argv is given to the main. you can write script_name.py like this::
...
def setup(args):
mysql_port, mysql_host, mysql_user, mysql_password, mysql_database, base_url, htdocs = args
.......
if __name__ == '__main__':
setup(sys.argv[1:])
- base_url: is the url of php software
- htdocs: is the path of www-data directory
- mysql_user, mysql_password, mysql_database, mysql_host: is the mariadb parameters
you can also use "import MySQLdb" if you want to access to database via your python script
\ No newline at end of file
slapos/recipe/boinc/__init__.py deleted 100644 → 0
View file @ 07bbdab1
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import subprocess
import pwd
import json
import signal
import zc.buildout
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational boinc architecture."""
def __init__(self, buildout, name, options):
#get current slapuser name
stat_info = os.stat(options['home'].strip())
options['user'] = pwd.getpwuid(stat_info.st_uid)[0]
url_base = options['url-base']
project = options['project'].strip()
root = options['installroot'].strip()
options['home_page'] = url_base + "/" + project
options['admin_page'] = url_base + "/" + project + "_ops/"
options['result_page'] = url_base + "/" + project + "_result/"
options['cronjob'] = os.path.join(root, project+'.cronjob')
return GenericBaseRecipe.__init__(self, buildout, name, options)
def _options(self, options):
#Path of boinc compiled package
self.package = options['boinc'].strip()
self.sourcedir = options['source'].strip()
self.home = options['home'].strip()
self.project = options['project'].strip()
self.fullname = options['fullname'].strip()
self.copyright = options['copyright'].strip()
self.installroot = options['installroot'].strip()
self.boinc_egg = os.path.join(self.package, 'lib/python2.7/site-packages')
self.developegg = options['develop-egg'].strip()
self.wrapperdir = options['wrapper-dir'].strip()
self.passwd = options['passwd'].strip()
#Get binary path
self.svn = options['svn-binary'].strip()
self.perl = options['perl-binary'].strip()
self.pythonbin = options['python-binary'].strip()
#Apache php informations
self.url_base =options['url-base'].strip()
self.htpasswd = options['htpasswd'].strip()
self.phpini = options['php-ini'].strip()
self.phpbin = options['php-bin'].strip()
#get Mysql parameters
self.username = options['mysql-username'].strip()
self.password = options['mysql-password'].strip()
self.database = options['mysql-database'].strip()
self.mysqlhost = options['mysql-host'].strip()
self.mysqlport = options['mysql-port'].strip()
def haschanges(self):
config_file = os.path.join(self.home, '.config')
current = [self.fullname, self.copyright,
self.password, self.mysqlhost, self.installroot,
self.project, self.passwd, self.url_base]
previous = []
result = False
if os.path.exists(config_file):
previous = open(config_file, 'r').read().split('#')
#Check if config has changed
if len(current) != len(set(current).intersection(set(previous))) or \
not os.path.exists(self.installroot) or \
not os.path.exists(os.path.join(self.home, '.start_service')):
result = True
open(config_file, 'w').write('#'.join(current))
return result
def install(self):
path_list = []
make_project = os.path.join(self.package, 'bin/make_project')
niceprojectname = self.project + "@Home"
slapuser = self.options['user']
#Check if given URL is not empty (case of URL request with frontend)
if not self.url_base:
raise Exception("URL_BASE is still empty. Can not use it")
#Define environment variable here
python = os.path.join(self.home, 'bin/python')
python_path = self.boinc_egg
if not os.path.exists(python):
os.symlink(self.pythonbin, python)
for f in os.listdir(self.developegg):
dir = os.path.join(self.developegg, f)
if os.path.isdir(dir):
python_path += ":" + dir
bin_dir = os.path.join(self.home, 'bin')
environment = dict(
PATH=os.pathsep.join([self.svn, bin_dir, self.perl, os.environ['PATH']]),
PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
)
#Generate wrapper for php
wrapperphp = os.path.join(self.home, 'bin/php')
php_wrapper = self.createWrapper(wrapperphp,
(self.phpbin, '-c', self.phpini),
)
path_list.append(php_wrapper)
mysql_dict = dict(db=self.database,
host=self.mysqlhost, port=self.mysqlport,
user=self.username, passwd=self.password)
#Generate python script for MySQL database test (starting)
file_status = os.path.join(self.home, '.boinc_config')
if os.path.exists(file_status):
os.unlink(file_status)
mysql_wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'start_config'),
'%s.configure.checkMysql' % __name__,
(environment, mysql_dict, file_status)
)
# Generate make project wrapper file
readme_file = os.path.join(self.installroot, self.project+'.readme')
launch_args = [make_project, '--url_base', self.url_base, "--db_name",
self.database, "--db_user", self.username, "--db_passwd",
self.password, "--project_root", self.installroot, "--db_host",
self.mysqlhost, "--user_name", slapuser, "--srcdir",
self.sourcedir, "--no_query"]
drop_install = self.haschanges()
request_make_boinc = os.path.join(self.home, '.make_project')
if drop_install:
#Allow to restart Boinc installation from the begining
launch_args += ["--delete_prev_inst", "--drop_db_first"]
open(request_make_boinc, 'w').write('Make Requested')
if os.path.exists(readme_file):
os.unlink(readme_file)
launch_args += [self.project, niceprojectname]
install_wrapper = self.createPythonScript(
os.path.join(self.wrapperdir, 'make_project'),
'%s.configure.makeProject' % __name__,
(file_status, launch_args, request_make_boinc, environment)
)
path_list.append(install_wrapper)
#generate sh script for project configuration
bash = os.path.join(self.home, 'bin', 'project_config.sh')
sh_script = self.createFile(bash,
self.substituteTemplate(self.getTemplateFilename('project_config.in'),
dict(dash=self.options['dash'].strip(),
uldl_pid=self.options['apache-pid'].strip(),
user=slapuser, fullname=self.fullname,
copyright=self.copyright, installroot=self.installroot))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
#After make_project run configure_script to perform and restart apache php services
service_status = os.path.join(self.home, '.start_service')
parameter = dict(
readme=readme_file,
htpasswd=self.htpasswd,
installroot=self.installroot,
username=slapuser,
passwd=self.passwd,
xadd=os.path.join(self.installroot, 'bin/xadd'),
environment=environment,
service_status=service_status,
drop_install=drop_install,
sedconfig=bash
)
start_service = self.createPythonScript(
os.path.join(self.wrapperdir, 'config_project'),
'%s.configure.services' % __name__, (parameter,)
)
path_list.append(start_service)
#Generate Boinc start project wrapper
start_args = [os.path.join(self.installroot, 'bin/start')]
start_boinc = os.path.join(self.home, '.start_boinc')
if os.path.exists(start_boinc):
os.unlink(start_boinc)
boinc_parameter = dict(service_status=service_status,
installroot=self.installroot, drop_install=drop_install,
mysql_dict=mysql_dict, environment=environment,
start_boinc=start_boinc)
start_wrapper = self.createPythonScript(os.path.join(self.wrapperdir,
'start_boinc'),
'%s.configure.restart_boinc' % __name__,
(boinc_parameter,)
)
path_list.append(start_wrapper)
return path_list
update = install
class App(GenericBaseRecipe):
"""This recipe allow to deploy an scientific applications using boinc
Note that recipe use depend on boinc-server parameter"""
def downloadFiles(self, app):
"""This is used to download app files if necessary and update options values"""
for key in ('input-file', 'template-result', 'template-wu', 'binary'):
param = app[key]
if param and (param.startswith('http') or param.startswith('ftp')):
#download the specified file
cache = os.path.join(self.options['home'].strip(), 'tmp')
downloader = zc.buildout.download.Download(self.buildout['buildout'],
hash_name=True, cache=cache)
path, _ = downloader(param, md5sum=None)
mode = 0600
if key == 'binary':
mode = 0700
os.chmod(path, mode)
app[key] = path
def getAppList(self):
"""Load parameters,
check if parameter send is valid to install or update application"""
app_list = json.loads(self.options['boinc-app-list'])
if not app_list:
return None
default_template_result = self.options.get('default-template-result', '').strip()
default_template_wu = self.options.get('default-template-wu', '').strip()
default_extension = self.options.get('default-extension', '').strip()
default_platform = self.options.get('default-platform', '').strip()
for app in app_list:
for version in app_list[app]:
current_app = app_list[app][version]
#Use default value if empty and Use_default is True
#Initialize all values to empty if not define by the user
if current_app['use_default']:
current_app['template-result'] = current_app.get('template-result',
default_template_result).strip()
current_app['template-wu'] = current_app.get('template-wu',
default_template_wu).strip()
current_app['extension'] = current_app.get('extension',
default_extension).strip()
current_app['platform'] = current_app.get('platform',
default_platform).strip()
else:
current_app['template-result'] = current_app.get('template-result', '').strip()
current_app['template-wu'] = current_app.get('template-wu', '').strip()
current_app['extension'] = current_app.get('extension', '').strip()
current_app['platform'] = current_app.get('platform', '').strip()
current_app['input-file'] = current_app.get('input-file', '').strip()
current_app['wu-number'] = current_app.get('wu-number', 1)
#for new application, check if parameter is complete
appdir = os.path.join(self.options['installroot'].strip(), 'apps',
app, version)
if not os.path.exists(appdir):
if not current_app['template-result'] or not current_app['binary'] \
or not current_app['input-file'] or not current_app['template-wu'] \
or not current_app['platform']:
print "BOINC-APP: ERROR - Invalid argements values for % ...operation cancelled" % app
app_list[app][version] = None
continue
#write application to install
request_file = os.path.join(self.options['home'].strip(),
'.install_' + app + version)
toInstall = open(request_file, 'w')
toInstall.write('install or update')
toInstall.close()
return app_list
def install(self):
app_list = self.getAppList()
path_list = []
package = self.options['boinc'].strip()
#Define environment variable here
developegg = self.options['develop-egg'].strip()
python_path = os.path.join(package, 'lib/python2.7/site-packages')
home = self.options['home'].strip()
user = pwd.getpwuid(os.stat(home).st_uid)[0]
perl = self.options['perl-binary'].strip()
svn = self.options['svn-binary'].strip()
for f in os.listdir(developegg):
dir = os.path.join(developegg, f)
if os.path.isdir(dir):
python_path += ":" + dir
bin_dir = os.path.join(home, 'bin')
environment = dict(
PATH=os.pathsep.join([svn, bin_dir, perl, os.environ['PATH']]),
PYTHONPATH=os.pathsep.join([python_path, os.environ['PYTHONPATH']]),
)
#generate project.xml and config.xml script updater
bash = os.path.join(home, 'bin', 'update_config.sh')
sh_script = self.createFile(bash,
self.substituteTemplate(self.getTemplateFilename('sed_update.in'),
dict(dash=self.options['dash'].strip()))
)
path_list.append(sh_script)
os.chmod(bash , 0700)
#If useful, download necessary files and update options path
start_boinc = os.path.join(home, '.start_boinc')
installroot = self.options['installroot'].strip()
apps_dir = os.path.join(installroot, 'apps')
wrapperdir = self.options['wrapper-dir'].strip()
project = self.options['project'].strip()
lockfile = os.path.join(self.options['home'].strip(), 'app_install.lock')
fd = os.open(lockfile, os.O_RDWR|os.O_CREAT)
os.close( fd )
for appname in app_list:
for version in app_list[appname]:
if not app_list[appname][version]:
continue
self.downloadFiles(app_list[appname][version])
platform = app_list[appname][version]['platform']
application = os.path.join(apps_dir, appname, version, platform)
if app_list[appname][version]['binary'] and not platform:
print "BOINC-APP: WARNING - Cannot specify binary without giving platform value"
app_list[appname][version]['binary'] = '' #Binary will not be updated
parameter = dict(installroot=installroot,
appname=appname, project=project,
version=version, platform=platform,
application=application, environment=environment,
start_boinc=start_boinc,
wu_number=app_list[appname][version]['wu-number'],
t_result=app_list[appname][version]['template-result'],
t_wu=app_list[appname][version]['template-wu'],
t_input=app_list[appname][version]['input-file'],
binary=app_list[appname][version]['binary'],
extension=app_list[appname][version]['extension'],
bash=bash, home_dir=home,
lockfile=lockfile,
)
deploy_app = self.createPythonScript(
os.path.join(wrapperdir, 'boinc_%s' % appname),
'%s.configure.deployApp' % __name__, (parameter,)
)
path_list.append(deploy_app)
return path_list
update = install
class Client(GenericBaseRecipe):
"""Deploy a fully fonctionnal boinc client connected to a boinc server instance"""
def __init__(self, buildout, name, options):
#get current uig to create a unique rpc-port for this client
stat_info = os.stat(options['home'].strip())
options['rpc-port'] = pwd.getpwuid(stat_info.st_uid)[2] + 5000
return GenericBaseRecipe.__init__(self, buildout, name, options)
def install(self):
path_list = []
boincbin = self.options['boinc-bin'].strip()
cmdbin = self.options['cmd-bin'].strip()
installdir = self.options['install-dir'].strip()
url = self.options['server-url'].strip()
key = self.options['key'].strip()
boinc_wrapper = self.options['client-wrapper'].strip()
cmd_wrapper = self.options['cmd-wrapper'].strip()
remote_host = os.path.join(installdir, 'remote_hosts.cfg')
open(remote_host, 'w').write(self.options['ip'].strip())
#Generate wrapper for boinc cmd
base_cmd = [cmdbin, '--host', str(self.options['rpc-port']),
'--passwd', self.options['passwd'].strip()]
cc_cmd = ''
if self.options['cconfig'].strip() != '':
config_dest = os.path.join(installdir, 'cc_config.xml')
file = open(config_dest, 'w')
file.write(open(self.options['cconfig'].strip(), 'r').read())
file.close()
cc_cmd = '--read_cc_config'
cmd = self.createPythonScript(cmd_wrapper,
'%s.configure.runCmd' % __name__,
(base_cmd, cc_cmd, installdir, url, key)
)
path_list.append(cmd)
#Generate BOINC client wrapper
boinc = self.createWrapper(boinc_wrapper,
(boincbin, '--allow_multiple_clients', '--gui_rpc_port',
str(self.options['rpc-port']), '--allow_remote_gui_rpc',
'--dir', installdir, '--redirectio', '--check_all_logins'),
)
path_list.append(boinc)
return path_list
slapos/recipe/boinc/configure.py deleted 100644 → 0
View file @ 07bbdab1
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import sys
import subprocess
import time
import shutil
import re
import filecmp
from lock_file import LockFile
def checkMysql(environment, connect_kw, file_status=None):
sys.path += environment['PYTHONPATH'].split(':')
import MySQLdb
#Sleep until mysql server becomes available
while True:
try:
MySQLdb.connect(**connect_kw).close()
break
except Exception, ex:
print "The result is: \n" + ex.message
print "Could not connect to MySQL database... sleep for 2 secondes"
time.sleep(2)
print "Successfully connect to MySQL database... "
if file_status:
writeFile(file_status, "starting")
def checkFile(file, stime):
"""Loop until 'file' is created (exist)"""
while True:
print "Search for file %s..." % file
if not os.path.exists(file):
print "File not found... sleep for %s secondes" % stime
time.sleep(stime)
else:
break
def restart_boinc(args):
"""Stop (if currently is running state) and start all Boinc service"""
environment = args['environment']
if args['drop_install']:
checkFile(args['service_status'], 3)
else:
checkMysql(environment, args['mysql_dict'], args.get('file_status'))
print "Restart Boinc..."
env = os.environ.copy()
env.update(environment)
subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
writeFile(args['start_boinc'], "started")
print "Done."
def check_installRequest(args):
print "Cheking if needed to install %s..." % args['appname']
install_request_file = os.path.join(args['home_dir'],
'.install_' + args['appname'] + args['version'])
if not os.path.exists(install_request_file):
print "No install or update request for %s version %s..." % (
args['appname'], args['version'])
return False
os.unlink(install_request_file)
return True
def copy_file(source, dest):
""""Copy file with source to dest with auto replace
return True if file has been copied and dest ha been replaced
"""
result = False
if source and os.path.exists(source):
if os.path.exists(dest):
if filecmp.cmp(dest, source):
return False
os.unlink(dest)
result = True
shutil.copy(source, dest)
return result
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env,
cwd=cwd)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
print "Failed to execute executable.\nThe error was: %s" % result
return False
return True
def makeProject(make_sig, launch_args, request_file, extra_environ):
"""Run BOINC make_project script but once only"""
#Wait for DateBase initialization...
checkFile(make_sig, 3)
print "Cheking if needed to run BOINC make_project..."
if os.path.exists(request_file):
env = os.environ.copy()
env.update(extra_environ)
if startProcess(launch_args, env=env):
os.unlink(request_file)
print "Finished running BOINC make_projet...Ending"
else:
print "No new request for make_project. Exiting..."
def services(args):
"""This function configure a new installed boinc project instance"""
print "Checking if needed to install or reinstall Boinc-server..."
if not args['drop_install']:
print "Not need to install Boinc-server...skipped"
return
#Sleep until file 'boinc_project'.readme exist
checkFile(args['readme'], 3)
topath = os.path.join(args['installroot'], 'html/ops/.htpasswd')
print "Generating .htpasswd file... File=%s" % topath
passwd = open(args['passwd'], 'r').read()
htpwd_args = [args['htpasswd'], '-b', '-c', topath, args['username'], passwd]
if not startProcess(htpwd_args):
return
print "execute script xadd..."
env = os.environ.copy()
env.update(args['environment'])
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return
print "Update files and directories permissions..."
upload = os.path.join(args['installroot'], 'upload')
inc = os.path.join(args['installroot'], 'html/inc')
languages = os.path.join(args['installroot'], 'html/languages')
compiled = os.path.join(args['installroot'], 'html/languages/compiled')
user_profile = os.path.join(args['installroot'], 'html/user_profile')
forum_file = os.path.join(args['installroot'], 'html/ops/create_forums.php')
project_inc = os.path.join(args['installroot'], 'html/project/project.inc')
cmd = "chmod 02700 -R %s %s, %s %s %s" % (upload, inc,
languages, compiled, user_profile)
os.system("chmod g+w -R " + args['installroot'])
os.system(cmd)
os.system("chmod 700 %s" % os.path.join(args['installroot'], 'keys'))
os.system("chmod o+x " + inc)
os.system("chmod -R o+r " + inc)
os.system("chmod o+x " + languages)
os.system("chmod o+x " + compiled)
sed_args = [args['sedconfig']]
startProcess(sed_args)
#Execute php create_forum.php...
print "Boinc Forum: Execute php create_forum.php..."
cwd = os.path.join(args['installroot'], 'html/ops')
if not startProcess(["php", forum_file], env, cwd):
return
writeFile(args['service_status'], "started")
def deployApp(args):
"""Deploy Boinc App with lock"""
print "Asking to enter in execution with lock mode..."
with LockFile(args['lockfile'], wait=True):
print "acquire the lock file..."
deployManagement(args)
print "Exit execution with lock..."
def deployManagement(args):
"""Fully deploy or redeploy or update a BOINC application using existing BOINC instance"""
if not check_installRequest(args):
return
token = os.path.join(args['installroot'], "." + args['appname'] + args['version'])
newInstall = False
if os.path.exists(token):
args['previous_wu'] = int(open(token, 'r').read().strip())
if args['previous_wu'] < args['wu_number']:
print args['appname'] + " Work units will be updated from %s to %s" % (
args['previous_wu'], args['wu_number'])
else:
args['previous_wu'] = 0
newInstall = True
#Sleep until file .start_boinc exist (File indicate that BOINC has been started)
checkFile(args['start_boinc'], 3)
env = os.environ.copy()
env.update(args['environment'])
print "setup directories..."
numversion = args['version'].replace('.', '')
args['inputfile'] = os.path.join(args['installroot'], 'download',
args['appname'] + numversion + '_input')
base_app = os.path.join(args['installroot'], 'apps', args['appname'])
base_app_version = os.path.join(base_app, args['version'])
args['templates'] = os.path.join(args['installroot'], 'templates')
t_result = os.path.join(args['templates'],
args['appname'] + numversion + '_result')
t_wu = os.path.join(args['templates'],
args['appname'] + numversion + '_wu')
binary_name = args['appname'] +"_"+ args['version'] +"_"+ \
args['platform'] + args['extension']
binary = os.path.join(args['application'], binary_name)
signBin = False
if not os.path.exists(base_app):
os.mkdir(base_app)
if newInstall:
if os.path.exists(base_app_version):
shutil.rmtree(base_app_version)
os.mkdir(base_app_version)
os.mkdir(args['application'])
if not os.path.exists(args['templates']):
os.mkdir(args['templates'])
copy_file(args['t_result'], t_result)
copy_file(args['t_wu'], t_wu)
signBin = copy_file(args['binary'], binary)
if args['t_input']:
if os.path.exists(args['inputfile']):
os.unlink(args['inputfile'])
os.symlink(args['t_input'], args['inputfile'])
project_xml = os.path.join(args['installroot'], 'project.xml')
findapp = re.search("<name>(%s)</name>" % args['appname'],
open(project_xml, 'r').read())
if not findapp:
print "Adding '" + args['appname'] + "' to project.xml..."
print "Adding deamon for application to config.xml..."
sed_args = [args['bash'], args['appname'], args['installroot']]
startProcess(sed_args)
if signBin:
print "Sign the application binary..."
sign = os.path.join(args['installroot'], 'bin/sign_executable')
privateKeyFile = os.path.join(args['installroot'], 'keys/code_sign_private')
output = open(binary + '.sig', 'w')
p_sign = subprocess.Popen([sign, binary, privateKeyFile], stdout=output,
stderr=subprocess.STDOUT, env=env)
result = p_sign.communicate()[0]
if p_sign.returncode is None or p_sign.returncode != 0:
print "Failed to execute bin/sign_executable.\nThe error was: %s" % result
return
output.close()
print "execute script xadd..."
if not startProcess([os.path.join(args['installroot'], 'bin/xadd')], env):
return
print "Running script bin/update_versions..."
updt_version = os.path.join(args['installroot'], 'bin/update_versions')
p_version = subprocess.Popen([updt_version], stdout=subprocess.PIPE,
stderr=subprocess.STDOUT, stdin=subprocess.PIPE, env=env,
cwd=args['installroot'])
p_version.stdin.write('y\ny\n')
result = p_version.communicate()[0]
p_version.stdin.close()
if p_version.returncode is None or p_version.returncode != 0:
print "Failed to execute bin/update_versions.\nThe error was: %s" % result
return
print "Fill the database... calling bin/create_work..."
create_wu(args, env)
print "Restart Boinc..."
subprocess.call((os.path.join(args['installroot'], 'bin', 'stop'),), env=env)
subprocess.call((os.path.join(args['installroot'], 'bin', 'start'),), env=env)
print "Boinc Application deployment is done... writing end signal file..."
writeFile(token, str(args['wu_number']))
def create_wu(args, env):
"""Create or update number of work unit for an existing boinc application"""
numversion = args['version'].replace('.', '')
t_result = "templates/" + args['appname'] + numversion + '_result'
t_wu = "templates/" + args['appname'] + numversion + '_wu'
launch_args = [os.path.join(args['installroot'], 'bin/create_work'),
'--appname', args['appname'], '--wu_name', '',
'--wu_template', t_wu, '--result_template', t_result,
'--min_quorum', '1', '--target_nresults', '1',
args['appname'] + numversion + '_input']
for i in range(args['previous_wu'], args['wu_number']):
print "Creating project wroker %s..." % str(i+1)
launch_args[4] = args['appname'] + str(i+1) + numversion + '_nodelete'
startProcess(launch_args, env, args['installroot'])
def runCmd(base_cmd, cc_cmd, installdir, url, key):
"""Wait for Boinc Client started and run boinc cmd"""
client_config = os.path.join(installdir, 'client_state.xml')
checkFile(client_config, 5)
time.sleep(10)
#Scan client state xml to find client ipv4 adress
host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1)
base_cmd[2] = host + ':' + base_cmd[2]
print "Run boinccmd with host at %s " % base_cmd[2]
project_args = base_cmd + ['--project_attach', url, key]
startProcess(project_args, cwd=installdir)
if cc_cmd:
#Load or reload cc_config file
startProcess(base_cmd + [cc_cmd], cwd=installdir)
def writeFile(file, content):
f = open(file, 'w')
f.write(content)
f.close()
slapos/recipe/boinc/template/project_config.in deleted 100644 → 0
View file @ 07bbdab1
#!%(dash)s
sed -i "/remove the die/d" %(installroot)s/html/ops/create_forums.php
sed -i "s#REPLACE WITH PROJECT NAME#%(fullname)s#" %(installroot)s/html/project/project.inc
sed -i "s#REPLACE WITH COPYRIGHT HOLDER#%(copyright)s#" %(installroot)s/html/project/project.inc
sed -i.old -e "/<\/config>/i\<uldl_pid>%(uldl_pid)s</uldl_pid>" %(installroot)s/config.xml
sed -i.old -e "/<\/config>/i\<httpd_user>%(user)s</httpd_user>" %(installroot)s/config.xml
slapos/recipe/boinc/template/sed_update.in deleted 100644 → 0
View file @ 07bbdab1
#!%(dash)s
if [ $# -ne 2 ]
then
echo not enough argument.
echo ARGS: appname installroot
else
sed -i.old -e "/<\/boinc>/i\<app>\n<name>$1</name>\n<user_friendly_name>$1</user_friendly_name>\n</app>" $2/project.xml
sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_bitwise_validator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml
sed -i.old -e "/<\/daemons>/i\<daemon>\n<cmd>\nsample_assimilator -d 3 -app $1\n</cmd>\n</daemon>" $2/config.xml
fi
\ No newline at end of file
slapos/recipe/bonjourgrid/__init__.py deleted 100644 → 0
View file @ 07bbdab1
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
from slapos.recipe.librecipe import GenericBaseRecipe
import os
import subprocess
import pwd
import signal
import zc.buildout
class Recipe(GenericBaseRecipe):
"""Deploy a fully operational BonjourGrid Master architecture."""
def install(self):
path_list = []
condor_wrapper = self.options['condor-wrapper'].strip()
boinc_wrapper = self.options['boinc-wrapper'].strip()
startCondor = startBoinc = False
condor_wrapper_list = []
boinc_wrapper_list = []
for file in os.listdir(condor_wrapper):
condor_wrapper_list.append(os.path.join(condor_wrapper, file))
for file in os.listdir(boinc_wrapper):
boinc_wrapper_list.append(os.path.join(boinc_wrapper, file))
type = self.options['type'].strip()
if type == "condor":
startCondor = True
project = self.options['condor_host'].strip()
if type == "boinc":
project = self.options['url-boinc'].strip()
startBoinc = True
#Generate BOINC/Condor launcher script
grid_wrapper = self.options['boinc_condor_wrapper'].strip()
parameters = dict(startCondor=startCondor, startBoinc=startBoinc,
bg_base=self.options['work_dir'].strip(),
condor_wrapper_list=condor_wrapper_list,
boinc_wrapper_list=boinc_wrapper_list)
bonjourGrid_wrapper = self.createPythonScript(grid_wrapper,
__name__ + '.configure.launchScript',
(parameters,)
)
path_list.append(bonjourGrid_wrapper)
#Generate wrapper for BonjourGrid Master
bonjourgrid_master = self.options['master_script'].strip()
python = self.options['python-bin'].strip()
bg_wrapper = self.options['wrapper'].strip()
log = self.options['log_file'].strip()
pid_file = self.options['pid_file'].strip()
wrapper = self.createWrapper(bg_wrapper,
(python, bonjourgrid_master, '--log_file', log,
'--pid_file', pid_file,
'--master_wrapper', grid_wrapper,
'--directory', self.options['work_dir'].strip(),
'--server', self.options['redis-url'].strip(),
'--port', self.options['redis-port'].strip(),
'--num_workers', self.options['nworkers'].strip(),
),
)
path_list.append(wrapper)
#generate Computer information file
config_info_file = os.path.join(self.options['work_dir'].strip(),
'machineinfo.sh')
config_info = self.createFile(config_info_file,
self.substituteTemplate(self.getTemplateFilename('machineinfo.sh.in'),
dict(ip_address=self.options['ipv6'].strip(),
project=project,
middleware=type)))
os.chmod(config_info_file, 0o744)
path_list.append(config_info)
update = install
class Client(GenericBaseRecipe):
def install(self):
path_list = []
boinc_script = self.options['boinc_script'].strip()
condor_script = self.options['condor_script'].strip()
#Generate wrapper for BonjourGrid Worker
bonjourgrid_client = self.options['client_script'].strip()
python = self.options['python-bin'].strip()
bg_wrapper = self.options['wrapper'].strip()
log = self.options['log_file'].strip()
pid_file = self.options['pid_file'].strip()
wrapper = self.createWrapper(bg_wrapper,
(python, bonjourgrid_client, '--log_file', log,
'--pid_file', pid_file,
'--boinc_wrapper', boinc_script,
'--condor_wrapper', condor_script,
'--directory', self.options['work_dir'].strip(),
'--install_directory', self.options['install_dir'].strip(),
'--server', self.options['redis-url'].strip(),
'--port', self.options['redis-port'].strip(),
),
)
path_list.append(wrapper)
#generate BOINC and Condor configure script for bonjourgrid
boinc_wrapper = self.createPythonScript(boinc_script,
'%s.boinc.runBoinc' % __name__,
dict(ipv6=self.options['ipv6'].strip(),
email=self.options['email'].strip(),
boinc_wrapper=self.options['boinc_wrapper'].strip(),
boinc_cmd=self.options['boinc_cmd'].strip(),
boinc_rpc_port=self.options['boinc_rpc_port'],
boinc_install_dir=self.options['boinc_install_dir'].strip(),
boinc_passwd=self.options['boinc_passwd'].strip(),
account_name=self.options['account_name'].strip(),
account_passwd=self.options['account_passwd'].strip(),
)
)
path_list.append(boinc_wrapper)
condor_wrapper = self.createPythonScript(condor_script,
'%s.condor.runCondor' % __name__,
dict(ipv6=self.options['ipv6'].strip(),
condor_bin=self.options['condor_bin'].strip(),
condor_sbin=self.options['condor_sbin'].strip(),
condor_wrapper=self.options['condor_wrapper'].strip(),
condor_config=self.options['condor_config'].strip(),
condor_config_local=self.options['condor_config_local'].strip(),
)
)
path_list.append(condor_wrapper)
slapos/recipe/bonjourgrid/boinc.py deleted 100644 → 0
View file @ 07bbdab1
# -*- coding: utf-8 -*-
import os
import sys
import re
import subprocess
import time
import signal
def startProcess(launch_args, env=None, cwd=None, stdout=subprocess.PIPE):
process = subprocess.Popen(launch_args, stdout=stdout,
stderr=subprocess.STDOUT, env=env,
cwd=cwd)
result = process.communicate()[0]
if process.returncode is None or process.returncode != 0:
raise NameError("Failed to execute executable.\nThe error was: %s" % result)
def joinProject(args, base_cmd):
"""Finish BOINC Client configuration with create account and attach project"""
project_args = base_cmd + ['--project_attach', args['project_url'],
args['key']]
startProcess(project_args, cwd=args['boinc_install_dir'])
def createAccount(config, base_cmd):
"""Connect to BOINC Master and create an account
"""
account_args = base_cmd + ['--create_account', config['project_url'],
config['email'], config['account_passwd'],
config['account_name']]
startProcess(account_args, cwd=config['boinc_install_dir'])
account_file = os.path.join(config['boinc_install_dir'], 'create_account.xml')
key = re.search("<authenticator>([\w\d\._]+)</authenticator>",
open(account_file, 'r').read()).group(1)
return key
def runBoinc(config):
if len(sys.argv) < 2:
print "Argument Error: uses %s project_url" % sys.argv[0]
exit(1)
if type(config) == type(""):
print "Error: bonjourgrid.xml parsing error, file not exist or corrupted"
exit(1)
#XXX Using define values here for Boinc Master URL
config['project_url'] = sys.argv[1]
#launch Boinc Client
boinc = subprocess.Popen([config['boinc_wrapper']],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
#Retrieve ipv4 using by Boinc-client in client-configuration
client_config = os.path.join(config['boinc_install_dir'], 'client_state.xml')
while not os.path.exists(client_config):
time.sleep(5)
print "Search for file '%r'..." % client_config
time.sleep(10)
try:
#Scan client state xml to find client ipv4 adress
host = re.search("<ip_addr>([\w\d\.:]+)</ip_addr>",
open(client_config, 'r').read()).group(1)
base_cmd = [config['boinc_cmd'], '--host',
host + ":" + str(config['boinc_rpc_port']),
'--passwd', config['boinc_passwd']]
#Create Account for current instance on BOINC master
print "Create account for current client..."
key = createAccount(config, base_cmd)
config['key'] = key
print "Done. The account key is %s" % key
#Attach project to Boinc Master
print "Attach client to Boinc Master at %s " % config['project_url']
try:
joinProject(config, base_cmd)
except Exception, e:
print e
print "Done! Waiting for Boinc Client now..."
except Exception, e:
#An error occure!!!
os.kill(boinc.pid, signal.SIGTERM)
print e
#wait for Boinc client execution
boinc.wait()
slapos/recipe/bonjourgrid/condor.py deleted 100644 → 0
View file @ 07bbdab1
# -*- coding: utf-8 -*-
import os
import sys
import re
import subprocess
import time
def writeFile(file, content):
f = open(file, 'w')
f.write(content)
f.close()
def updateCondorConfig(path, path_local, hostname, ipv6):
"""Replace Static information into condor_config and condor_config.local files"""
#replace condor host into condor_config
text = open(path, 'r').read()
text = re.sub(r'\[%s\]' % ipv6, hostname, text, flags=re.IGNORECASE)
writeFile(path, text)
#replace condor host into condor_config.local
text = open(path_local, 'r').read()
text = re.sub(r'\[%s\]' % ipv6, hostname, text, flags=re.IGNORECASE)
writeFile(path_local, text)
def updateCondorWrapper(folder, hostname, ipv6):
"""Replace slapos generated value by the true value"""
for file in os.listdir(folder):
path = os.path.join(folder, file)
if os.path.exists(path) and not os.path.isdir(path):
text = re.sub(r'\[%s\]' % ipv6, hostname, open(path, 'r').read(),
flags=re.IGNORECASE)
writeFile(path, text)
def runCondor(config):
if len(sys.argv) < 2:
print "Argument Error: uses %s hostname" % sys.argv[0]
exit(1)
hostname = sys.argv[1]
updateCondorConfig(config['condor_config'], config['condor_config_local'],
hostname, config['ipv6'])
updateCondorWrapper(config['condor_bin'], hostname, config['ipv6'])
updateCondorWrapper(config['condor_sbin'], hostname, config['ipv6'])
#launch Boinc Client
condor = subprocess.Popen([config['condor_wrapper']],
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT)
condor.wait()
slapos/recipe/bonjourgrid/configure.py deleted 100644 → 0
View file @ 07bbdab1
##############################################################################
#
# Copyright (c) 2010 Vifib SARL and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import os
import subprocess
import time
def runProcess(args, file):
filename = file.split('/')[-1:][0]
stdout_file = open(os.path.join(args['bg_log'], filename + ".log"), 'w')
process = subprocess.Popen(file, stdout=stdout_file,
stderr=subprocess.STDOUT)
fp = open(os.path.join(args['bg_pid'], filename + '.pid'), 'w')
fp.write(str(process.pid))
fp.close()
return process.pid
def launchScript(args):
print "Sleep for a few second..."
time.sleep(10)
pid_list = []
bg_pid = os.path.join(args['bg_base'], 'pid')
bg_log = os.path.join(args['bg_base'], 'log')
args['bg_pid'] = bg_pid
args['bg_log'] = bg_log
if not os.path.exists(bg_pid):
os.mkdir(bg_pid)
if not os.path.exists(bg_log):
os.mkdir(bg_log)
#launch all Condor wrapper
if args['startCondor']:
for file in args['condor_wrapper_list']:
pid_list.append(runProcess(args, file))
#Launch all BOINC wrapper
if args['startBoinc']:
for file in args['boinc_wrapper_list']:
pid_list.append(runProcess(args, file))
for pid in pid_list:
print "Parent waiting for process child: %s " % pid
result = os.waitpid(pid, 0)
print "Done...", result
slapos/recipe/bonjourgrid/template/machineinfo.sh.in deleted 100755 → 0
View file @ 07bbdab1
# This script generates a file (with the name `hostname`) of the characteristics of the machine
Mhz=`grep -i Mhz /proc/cpuinfo | cut -d : -f2 | head -1 | cut -f2 -d " "`
mem=`grep -i MemTotal /proc/meminfo | cut -f2 -d :`
Ram=`echo $mem | cut -f1 -d " "`
Cpu=`uname -a | cut -f3 -d " "`
H=`hostname`
Nbproc=`grep processor /proc/cpuinfo |wc -l`
Ip=%(ip_address)s
status="in execution"
secours="host1"
middleware="%(middleware)s"
project="%(project)s"
c="{'"MHZ"':\"$Mhz\",'"Ram"':\"$Ram\",'"Cpu"':\"$Cpu\",'"HOST"':\"$H\",'"IP"':\"$Ip\",'"STATUS"':\"$status\",'"SECOURS"':\"$secours\",'"MIDDLEWARE"':\"$middleware\",'"PROJECT"':\"$project\",}"
filename=`hostname`
echo $c > $filename
software/backupserver/software.cfg
View file @ eb1f843f
......@@ -107,6 +107,3 @@ gunicorn = 19.1.1
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.3
PyRSS2Gen = 1.1
gitdb2 = 2.0.5
smmap2 = 2.0.5
software/caddy-frontend/buildout.hash.cfg
View file @ eb1f843f
......@@ -54,7 +54,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
[template-default-slave-virtualhost]
filename = templates/default-virtualhost.conf.in
md5sum = b5447e33658b8a81b75275630f9da119
md5sum = 7e26935bb6daf00d8fc01d97eebc7abd
[template-cached-slave-virtualhost]
filename = templates/cached-virtualhost.conf.in
......
software/caddy-frontend/instance-slave-caddy-input-schema.json
View file @ eb1f843f
......@@ -44,7 +44,7 @@
"type": "string"
},
"https-only": {
"default": "false",
"default": "true",
"description": "If set to true, http requests will be redirected to https",
"enum": [
"false",
......
software/caddy-frontend/instance-slave-caddy-simplified-input-schema.json
View file @ eb1f843f
......@@ -72,7 +72,7 @@
"type": "string"
},
"https-only": {
"default": "false",
"default": "true",
"description": "If set to true, http requests will be redirected to https",
"enum": [
"false",
......
software/caddy-frontend/templates/default-virtualhost.conf.in
View file @ eb1f843f
......@@ -10,7 +10,7 @@
{%- set enable_h2 = slave_parameter['global_disable_http2'].lower() not in TRUE_VALUES and slave_parameter.get('enable-http2', slave_parameter['enable_http2_by_default']).lower() in TRUE_VALUES %}
{%- set ssl_proxy_verify = slave_parameter.get('ssl-proxy-verify', '').lower() in TRUE_VALUES %}
{%- set disabled_cookie_list = slave_parameter.get('disabled-cookie-list', '').split() %}
{%- set https_only = slave_parameter.get('https-only', '').lower() in TRUE_VALUES %}
{%- set https_only = slave_parameter.get('https-only', 'true').lower() in TRUE_VALUES %}
{%- set slave_type = slave_parameter.get('type', '') %}
{%- set host_list = server_alias_list %}
{%- set cipher_list = slave_parameter.get('cipher_list', '').strip() %}
......
software/caddy-frontend/test/test.py
View file @ eb1f843f
......@@ -1163,7 +1163,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
},
'https-only': {
'url': cls.backend_url,
'https-only': True,
'https-only': False,
},
'custom_domain': {
'url': cls.backend_url,
......@@ -1208,7 +1208,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
'url': cls.backend_url,
'prefer-gzip-encoding-to-backend': 'true',
'type': 'zope',
'https-only': 'true',
'https-only': 'false',
},
'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt': {
'url': cls.backend_https_url,
......@@ -1230,7 +1230,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
'type-zope-virtualhostroot-http-port': {
'url': cls.backend_url,
'type': 'zope',
'virtualhostroot-http-port': '12345'
'virtualhostroot-http-port': '12345',
'https-only': 'false',
},
'type-zope-virtualhostroot-https-port': {
'url': cls.backend_url,
......@@ -1364,7 +1365,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
'prefer-gzip-encoding-to-backend-https-only': {
'url': cls.backend_url,
'prefer-gzip-encoding-to-backend': 'true',
'https-only': 'true',
'https-only': 'false',
},
'disabled-cookie-list': {
'url': cls.backend_url,
......@@ -1624,7 +1625,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
log_regexp)
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://empty.example.com/test-path',
result_http.headers['Location']
)
# check that 404 is as configured
result_missing = fakeHTTPSResult(
......@@ -1693,19 +1702,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(result_http, 'Path', '/test-path/deeper')
try:
j = result_http.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertFalse('remote_user' in j['Incoming Headers'].keys())
self.assertFalse('Content-Encoding' in result_http.headers)
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'secured=value;secure, nonsecured=value',
result_http.headers['Set-Cookie']
'https://url.example.com/test-path/deeper',
result_http.headers['Location']
)
# check that try_duration == 5 in the test_url slave
......@@ -2180,15 +2185,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://httpsonly.example.com/test-path/deeper',
result_http.headers['Location']
)
self.assertEqualResultJson(result_http, 'Path', '/test-path/deeper')
def test_custom_domain(self):
reference = 'custom_domain'
......@@ -2350,16 +2347,19 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/http//typezope.example.com:80/'
'/VirtualHostRoot/test-path/deeper'
self.assertEqual(
httplib.FOUND,
result.status_code
)
def test_type_zope_prefer_gzip_encoding_to_backend(self):
self.assertEqual(
'https://typezope.example.com/test-path/deep/.././deeper',
result.headers['Location']
)
def test_type_zope_prefer_gzip_encoding_to_backend_https_only(self):
parameter_dict = self.assertSlaveBase(
'type-zope-prefer-gzip-encoding-to-backend')
'type-zope-prefer-gzip-encoding-to-backend-https-only')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
......@@ -2379,7 +2379,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result,
'Path',
'/VirtualHostBase/https//'
'typezopeprefergzipencodingtobackend.example.com:443/'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
......@@ -2391,7 +2391,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result,
'Path',
'/VirtualHostBase/http//'
'typezopeprefergzipencodingtobackend.example.com:80/'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:80/'
'/VirtualHostRoot/test-path/deeper'
)
......@@ -2414,7 +2414,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result,
'Path',
'/VirtualHostBase/https//'
'typezopeprefergzipencodingtobackend.example.com:443/'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
self.assertEqual(
......@@ -2429,15 +2429,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result,
'Path',
'/VirtualHostBase/http//'
'typezopeprefergzipencodingtobackend.example.com:80/'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:80/'
'/VirtualHostRoot/test-path/deeper'
)
self.assertEqual(
'gzip', result.json()['Incoming Headers']['accept-encoding'])
def test_type_zope_prefer_gzip_encoding_to_backend_https_only(self):
def test_type_zope_prefer_gzip_encoding_to_backend(self):
parameter_dict = self.assertSlaveBase(
'type-zope-prefer-gzip-encoding-to-backend-https-only')
'type-zope-prefer-gzip-encoding-to-backend')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
......@@ -2457,7 +2457,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result,
'Path',
'/VirtualHostBase/https//'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
'typezopeprefergzipencodingtobackend.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
......@@ -2494,7 +2494,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result,
'Path',
'/VirtualHostBase/https//'
'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
'typezopeprefergzipencodingtobackend.example.com:443/'
'/VirtualHostRoot/test-path/deeper'
)
self.assertEqual(
......@@ -2889,10 +2889,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.BAD_GATEWAY,
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://sslproxyverifysslproxycacrtunverified.example.com/test-path',
result_http.headers['Location']
)
def test_ssl_proxy_verify_ssl_proxy_ca_crt(self):
parameter_dict = self.assertSlaveBase('ssl-proxy-verify_ssl_proxy_ca_crt')
......@@ -2920,19 +2925,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqualResultJson(result_http, 'Path', '/test-path')
try:
j = result_http.json()
except Exception:
raise ValueError('JSON decode problem in:\n%s' % (result.text,))
self.assertFalse('remote_user' in j['Incoming Headers'].keys())
self.assertFalse('Content-Encoding' in result.headers)
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'secured=value;secure, nonsecured=value',
result_http.headers['Set-Cookie']
'https://sslproxyverifysslproxycacrt.example.com/test-path',
result_http.headers['Location']
)
@skip('Not implemented in new test system')
......@@ -2999,10 +3000,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
'test-path/deeper')
self.assertEqual(
httplib.BAD_GATEWAY,
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://enablecachesslproxyverifysslproxycacrtunverified.example.com/'
'test-path/deeper',
result_http.headers['Location']
)
def test_enable_cache_ssl_proxy_verify_ssl_proxy_ca_crt(self):
parameter_dict = self.assertSlaveBase(
'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt')
......@@ -3131,10 +3138,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(
httplib.BAD_GATEWAY,
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://typezopesslproxyverifysslproxycacrtunverified.example.com/'
'test-path',
result_http.headers['Location']
)
def test_type_zope_ssl_proxy_verify_ssl_proxy_ca_crt(self):
parameter_dict = self.assertSlaveBase(
'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt')
......@@ -3163,12 +3176,14 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqualResultJson(
result,
'Path',
'/VirtualHostBase/http//'
'typezopesslproxyverifysslproxycacrt.example.com:80/'
'/VirtualHostRoot/test-path'
self.assertEqual(
httplib.FOUND,
result.status_code
)
self.assertEqual(
'https://typezopesslproxyverifysslproxycacrt.example.com/test-path',
result.headers['Location']
)
def test_type_zope_ssl_proxy_verify_unverified(self):
......@@ -3201,7 +3216,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://monitoripv6test.example.com/test-path',
result_http.headers['Location']
)
monitor_file = glob.glob(
os.path.join(
......@@ -3230,7 +3253,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://monitoripv4test.example.com/test-path',
result_http.headers['Location']
)
monitor_file = glob.glob(
os.path.join(
......@@ -3260,7 +3291,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://re6stoptimaltest.example.com/test-path',
result_http.headers['Location']
)
monitor_file = glob.glob(
os.path.join(
......@@ -3290,7 +3329,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://ciphers.example.com/test-path',
result_http.headers['Location']
)
configuration_file = glob.glob(
os.path.join(
......@@ -3319,7 +3367,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
parameter_dict
)
result = fakeHTTPResult(
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
......@@ -3360,7 +3408,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
def test_enable_cache_server_alias(self):
parameter_dict = self.assertSlaveBase('enable_cache_server_alias')
result = fakeHTTPResult(
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
......@@ -3403,43 +3451,20 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
headers = result.headers.copy()
self.assertKeyWithPop('Server', headers)
self.assertKeyWithPop('Date', headers)
self.assertKeyWithPop('Age', headers)
# drop keys appearing randomly in headers
headers.pop('Transfer-Encoding', None)
headers.pop('Content-Length', None)
headers.pop('Connection', None)
headers.pop('Keep-Alive', None)
self.assertEqual(
{
'Content-type': 'application/json',
'Set-Cookie': 'secured=value;secure, nonsecured=value',
'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
'stale-if-error=3600'
},
headers
httplib.FOUND,
result.status_code
)
backend_headers = result.json()['Incoming Headers']
via = backend_headers.pop('via', None)
self.assertNotEqual(via, None)
self.assertRegexpMatches(
via,
r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.6\)$'
self.assertEqual(
'https://enablecacheserveralias1.example.com/test-path/deeper',
result.headers['Location']
)
def test_enable_cache(self):
parameter_dict = self.assertSlaveBase('enable_cache')
result = fakeHTTPResult(
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
......@@ -3499,8 +3524,12 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
'revalidate=3600, stale-if-error=3600'})
result = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deeper', # simple path, as ATS can't change them
# append with :HTTP_PORT to mimic access in ATS
parameter_dict['domain'] + ':' + HTTPS_PORT,
parameter_dict['public-ipv4'],
# prepend with HTTPS to mimic access via https in ATS
# use simple path, as it is changed in Caddy
'HTTPS/test-path/deeper',
port=23432, headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
'revalidate=3600, stale-if-error=3600'})
......@@ -3588,7 +3617,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
parameter_dict = self.assertSlaveBase('enable_cache')
# check that timeout seen by ATS does not result in many queries done
# to the backend and that next request works like a charm
result = fakeHTTPResult(
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test_enable_cache_ats_timeout', headers={
'Timeout': '15',
......@@ -3655,7 +3684,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
self.assertIn(matching_line_amount, [0, 1])
# the result is available immediately after
result = fakeHTTPResult(
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper', headers={
'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
......@@ -3819,8 +3848,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
self.assertTrue(
isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
def test_prefer_gzip_encoding_to_backend(self):
parameter_dict = self.assertSlaveBase('prefer-gzip-encoding-to-backend')
def test_prefer_gzip_encoding_to_backend_https_only(self):
parameter_dict = self.assertSlaveBase(
'prefer-gzip-encoding-to-backend-https-only')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
......@@ -3894,9 +3924,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
def test_prefer_gzip_encoding_to_backend_https_only(self):
def test_prefer_gzip_encoding_to_backend(self):
parameter_dict = self.assertSlaveBase(
'prefer-gzip-encoding-to-backend-https-only')
'prefer-gzip-encoding-to-backend')
result = fakeHTTPSResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
......@@ -4212,7 +4242,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'],
'test-path/deep/.././deeper')
self.assertEqualResultJson(result_http, 'Path', '/http/test-path/deeper')
self.assertEqual(
httplib.FOUND,
result_http.status_code
)
self.assertEqual(
'https://urlhttpsurl.example.com/test-path/deeper',
result_http.headers['Location']
)
@skip('Impossible to instantiate cluster with stopped partition')
......@@ -6601,7 +6640,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.OK, result_http.status_code)
self.assertEqual(httplib.FOUND, result_http.status_code)
configuration_file = glob.glob(
os.path.join(
......@@ -6627,7 +6666,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
result_http = fakeHTTPResult(
parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
self.assertEqual(httplib.OK, result_http.status_code)
self.assertEqual(httplib.FOUND, result_http.status_code)
configuration_file = glob.glob(
os.path.join(
......
software/htmlvalidatorserver/software.cfg
View file @ eb1f843f
......@@ -52,6 +52,3 @@ gunicorn = 19.1.1
plone.recipe.command = 1.1
slapos.recipe.template = 2.4.3
inotifyx = 0.2.2
gitdb2 = 2.0.3
smmap2 = 2.0.3
software/jupyter/software.cfg
View file @ eb1f843f
[buildout]
versions = versions
extends =
buildout.hash.cfg
../../stack/slapos.cfg
......@@ -142,10 +141,6 @@ scipy = 0.19.0
# tornado==4.4.2
singledispatch = 3.4.0.3
# Required by:
# matplotlib==2.1.2
subprocess32 = 3.2.7
# Required by:
# prompt-toolkit==1.0.13
wcwidth = 0.1.7
......
software/lamp-template/software.cfg
View file @ eb1f843f
......@@ -2,7 +2,6 @@
extends =
../../stack/lamp/buildout.cfg
../../stack/slapos.cfg
[application]
recipe = slapos.recipe.build:download-unpacked
......
software/neoppod/software-common.cfg
View file @ eb1f843f
......@@ -144,7 +144,6 @@ ZODB = 4.4.5
coverage = 4.5.1
mock = 3.0.5
ecdsa = 0.13
gitdb2 = 2.0.0
msgpack = 0.5.6
msgpack-python = 0.5.6
mysqlclient = 1.3.12
......@@ -153,7 +152,6 @@ pycrypto = 2.6.1
pycurl = 7.43.0
setproctitle = 1.1.10
slapos.recipe.template = 4.4
smmap2 = 2.0.1
transaction = 1.7.0
zodbpickle = 1.0.4
zodbtools = 0.0.0.dev4
......
software/nginx-push-stream/software.cfg
View file @ eb1f843f
......@@ -44,5 +44,3 @@ mode = 0644
[versions]
slapos.recipe.template = 4.4
inotifyx = 0.2.2
gitdb2 = 2.0.3
smmap2 = 2.0.3
software/powerdns/software.cfg
View file @ eb1f843f
......@@ -63,6 +63,3 @@ GitPython = 2.1.11
lockfile = 0.12.2
feedparser = 5.2.1
atomize = 0.2.0
gitdb2 = 2.0.4
smmap2 = 2.0.4
software/slapos-testing/software.cfg
View file @ eb1f843f
......@@ -201,7 +201,6 @@ zc.lockfile = 1.4
bcrypt = 3.1.4
dnspython = 1.15.0
funcsigs = 1.0.2
gitdb2 = 2.0.4
httmock = 1.2.6
manuel = 1.9.0
mock = 2.0.0
......@@ -209,7 +208,6 @@ testfixtures = 6.11.0
pem = 18.1.0
pycurl = 7.43.0.2
pyflakes = 2.0.0
smmap2 = 2.0.4
zope.testing = 4.6.2
urllib3 = 1.24.1
# Required by:
......
software/slaprunner/README.rst
View file @ eb1f843f
......@@ -139,17 +139,6 @@ Example :
<parameter id="custom-frontend-basic-auth">true</parameter>
</instance>
Git repositories :
------------------
It is easy to give access to your git repository/ies to everyone, or to clone it on your own computer. For this, fetch the git_url parameter returned by your webrunner instance. It should look like that : https://[IPV6]:PORT/git/YourRepo.git/
To create the repo, go to the folder srv/runner/project and initiate a new git repo (git init/clone --bare YourRepo). Then, to clone the repo, just append its name to the previous URL.
Please notice that your repositories are in public access for reading. However, an account is needed to push to the repository (use your webrunner account, or create a new user in the parameters section of the webrunner).
For the moment, the PORT is the port of monitoring, which is 9685.
Things to notice for the nex developer :
----------------------------------------
......
software/slaprunner/buildout.hash.cfg
View file @ eb1f843f
......@@ -18,11 +18,11 @@ md5sum = 8d6878ff1d2e75010c50a1a2b0c13b24
[template-runner]
filename = instance-runner.cfg
md5sum = 7100c6678576c285322f1445029cd4f2
md5sum = 8d85d8d80f1ae3ff13946d14d0d8fe72
[template-runner-import-script]
filename = template/runner-import.sh.jinja2
md5sum = 2852689640b632fc713d7a603ce66207
md5sum = f2e2493bc5da90a53f86e5bcf64d2d57
[instance-runner-import]
filename = instance-runner-import.cfg.in
......@@ -34,7 +34,7 @@ md5sum = b992bb3391de9d6d422bfa8011d8ffc4
[template-resilient]
filename = instance-resilient.cfg.jinja2
md5sum = 105ed7f54f251c64a2d34559360a5604
md5sum = bd0ad0b80d2b39189f9665c48f1b3830
[template_nginx_conf]
filename = nginx_conf.in
......@@ -42,7 +42,7 @@ md5sum = 2b06f7eb9a1d45d250d4b92a944db925
[template_httpd_conf]
filename = httpd_conf.in
md5sum = 40825ef2d259ae3b86dfd2f28f597d3a
md5sum = 1d96c29b931383b2a87b9ae049d07d74
[template_launcher]
filename = launcher.in
......@@ -50,7 +50,11 @@ md5sum = 525e37ea8b2acf6209869999b15071a6
[template-slapos-cfg]
filename = template/slapos.cfg.in
md5sum = e6a3ca1604ae5458248135cd6de0f3e6
md5sum = 95de0677e78fc06cc8304cc6caea9169
[template-slapformat-definition.cfg]
filename = template/slapformat-definition.cfg.in
md5sum = a94299dd2133956060486609e30b97ec
[template-parameters]
filename = parameters.xml.in
......@@ -62,7 +66,7 @@ md5sum = 7645048216fcf957f7773534cd0408dc
[template-supervisord]
filename = template/supervisord.conf.in
md5sum = 28f69b57c2835bddfcfbe312563eef51
md5sum = 37f053d75752e998fc3bb9e4bf29d776
[template-listener-slapgrid]
filename = template/listener_slapgrid.py.in
......
software/slaprunner/httpd_conf.in
View file @ eb1f843f
......@@ -96,33 +96,3 @@ Alias /share {{ parameters.runner_home }}
</LimitExcept>
</Directory>
SetEnv GIT_HTTP_EXPORT_ALL
ScriptSock {{ parameters.cgid_sock }}
ScriptAlias /git/ {{ parameters.git_http_backend }}/
ScriptAlias /git-public/ {{ parameters.git_http_backend }}/
RewriteCond %{QUERY_STRING} service=git-receive-pack [OR]
RewriteCond %{REQUEST_URI} /git-receive-pack$
<LocationMatch "^/git/">
SetEnv GIT_PROJECT_ROOT {{ parameters.project_private_folder }}
Order Deny,Allow
Deny from env=AUTHREQUIRED
AuthType Basic
AuthName "Git Access"
AuthUserFile "{{ parameters.htpasswd_file }}"
Require valid-user
</LocationMatch>
<LocationMatch "^/git-public/">
SetEnv GIT_PROJECT_ROOT {{ parameters.project_public_folder }}
Order Deny,Allow
Deny from env=AUTHREQUIRED
AuthType Basic
AuthName "Git Access"
AuthUserFile "{{ parameters.htpasswd_file }}"
Require valid-user
Satisfy any
</LocationMatch>
software/slaprunner/instance-resilient.cfg.jinja2
View file @ eb1f843f
......@@ -12,7 +12,7 @@
{% set number_of_instances = 2 %}
{% endif %}
{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'resilient-ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-url', 'ssh-command', 'webdav-url', 'public-url', 'git-public-url', 'git-private-url'] -%}
{% set slaprunner_return = ['init-user', 'init-password', 'url', 'ssh-public-key', 'resilient-ssh-url', 'notification-id', 'ip', 'backend-url', 'url', 'ssh-url', 'ssh-command', 'webdav-url', 'public-url'] -%}
{% set monitor_return = ['monitor-base-url', 'monitor-url', 'monitor-user', 'monitor-password'] -%}
{% set monitor_parameter = {'monitor-cors-domains': slapparameter_dict.pop('monitor-cors-domains', "monitor.app.officejs.com")} -%}
{% set monitor_dict = {'parameter': monitor_parameter, 'return': monitor_return, 'set-monitor-url': True} -%}
......@@ -72,8 +72,6 @@ init-password = ${publish-early:init-password}
ssh-command = ${request-runner:connection-ssh-command}
webdav-url = ${request-runner:connection-webdav-url}
public-url = ${request-runner:connection-public-url}
git-public-url = ${request-runner:connection-git-public-url}
git-private-url = ${request-runner:connection-git-private-url}
{% if slapparameter_dict.get('custom-frontend-backend-url') -%}
custom-frontend-url = ${request-runner:connection-custom-frontend-url}
{% endif %}
......
software/slaprunner/instance-runner-output-schema.json
View file @ eb1f843f
......@@ -29,14 +29,6 @@
"public_url": {
"description": "Url of the public folder of your runner",
"type": "string"
},
"git_public_url": {
"description": "Public url of your repositories.",
"type": "string"
},
"git_private_url": {
"description": "Private url for your repositories. You can use it to push.",
"type": "string"
}
},
"type": "object"
......
software/slaprunner/instance-runner.cfg
View file @ eb1f843f
......@@ -21,6 +21,7 @@ common-runner-parts =
shellinabox
shellinabox-service
slapos-cfg
slapformat-definition.cfg
cron-entry-prepare-software
deploy-instance-parameters
instance-software
......@@ -151,8 +152,6 @@ project-test = $${:test}/project
software-test = $${:test}/software
instance-test = $${:test}/instance
sessions = $${buildout:directory}/.sessions
private-project = $${:home}/.git-private
public-project = $${:home}/.git-public
[slaprunner]
slaprunner = ${buildout:bin-directory}/slaprunner
......@@ -162,6 +161,7 @@ supervisor = ${buildout:bin-directory}/slapgrid-supervisorctl
git-binary = ${git:location}/bin/git
root_check = false
slapos.cfg = $${directory:etc}/slapos.cfg
slapformat-definition.cfg = $${directory:etc}/slapformat-definition.cfg
working-directory = $${runnerdirectory:home}
project-directory = $${runnerdirectory:project}
instance_root = $${runnerdirectory:instance-root}
......@@ -389,11 +389,7 @@ htpasswd_file = $${directory:etc}/.htpasswd
etc_dir = $${directory:etc}
var_dir = $${directory:var}
project_folder = $${directory:project}
project_private_folder = $${runnerdirectory:private-project}
project_public_folder = $${runnerdirectory:public-project}
runner_home = $${runnerdirectory:home}
git_http_backend = ${git:location}/libexec/git-core/git-http-backend
cgid_sock = $${directory:run}/cgid.sock
httpd_cors_file = $${slaprunner-httpd-cors:location}
[httpd-conf]
......@@ -591,8 +587,6 @@ init-user = $${runner-htpasswd:user}
init-password = $${runner-htpasswd:password}
ssh-command = ssh $${user-info:pw-name}@$${slap-network-information:global-ipv6} -p $${runner-sshd-port:port}
ssh-url = ssh://$${user-info:pw-name};fingerprint=$${runner-sshd-publickey-fingerprint:fingerprint}@[$${slap-network-information:global-ipv6}]:$${runner-sshd-port:port}
git-public-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
git-private-url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
monitor-base-url = $${monitor-publish-parameters:monitor-base-url}
{% if slapparameter_dict.get('no-ipv4-frontend', 'false') == 'false' -%}
url = https://$${request-frontend:connection-domain}
......@@ -672,6 +666,17 @@ mode = 700
context =
section slaprunner test-runner
[slapformat-definition.cfg]
recipe = slapos.recipe.template:jinja2
template = ${template-slapformat-definition.cfg:location}/${template-slapformat-definition.cfg:filename}
rendered = $${slaprunner:slapformat-definition.cfg}
mode = 700
context =
section slaprunner slaprunner
key partition_user user-info:pw-name
import codecs codecs
raw buildout_shared_part_list_dump $${slaprunner:buildout-shared-part-list-dump}
[shellinabox]
recipe = slapos.recipe.template:jinja2
# We cannot use slapos.cookbook:wrapper here because this recipe escapes too much
......@@ -785,6 +790,10 @@ slapproxy-autostart = true
slapproxy-startsecs = 1
slapproxy-command = $${slaprunner:slapos} proxy start --logfile $${:slapproxy-log} --cfg $${:slapos-cfg}
slapproxy-log = $${directory:log}/slapproxy.log
slapformat = slapformat
slapformat-command = $${slaprunner:slapos} node format --cfg $${:slapos-cfg} --verbose --logfile $${:slapformat-log} --now -i $${:slapformat-definition.cfg}
slapformat-log = $${directory:log}/slapos-node-format.log
slapformat-startretries = 0
socket_name = unix://$${:socket_path}
socket_path = $${directory:tmp}/supervisord.sock
startsecs = 0
......@@ -792,6 +801,7 @@ startsecs = 0
# Processes should handle their logs by themselves
stderr_logfile = $${directory:log}/supervisord-errors.log
slapos-cfg = $${slaprunner:slapos.cfg}
slapformat-definition.cfg = $${slaprunner:slapformat-definition.cfg}
[supervisord-conf]
recipe = slapos.recipe.template:jinja2
......
software/slaprunner/software.cfg
View file @ eb1f843f
......@@ -42,6 +42,7 @@ common-parts =
instance-runner-import
instance-runner-export
template-slapos-cfg
template-slapformat-definition.cfg
template-slapuser-script
# XXX: we have to manually add this for resilience
rdiff-backup
......@@ -110,6 +111,10 @@ recipe = slapos.recipe.build:download
< = template-download-base
filename = slapos.cfg.in
[template-slapformat-definition.cfg]
< = template-download-base
filename = slapformat-definition.cfg.in
[template-parameters]
< = download-only-base
......
software/slaprunner/template/runner-import.sh.jinja2
View file @ eb1f843f
......@@ -173,7 +173,7 @@ if 1:
# We want to replace the last occurence only
old_software_release, partition = sys.argv[1], sys.argv[2]
for match in re.finditer("(slappart|test0-)[0-9][0-9]*", old_software_release):
for match in re.finditer("(slappart|test0-|s)[0-9][0-9]*", old_software_release):
start, end = match.start(), match.end()
print old_software_release[:start] + partition + old_software_release[end:]
......
software/slaprunner/template/slapformat-definition.cfg.in 0 → 100644
View file @ eb1f843f
[computer]
# Address example: 2001:67c:1254:27::1/ffff:ffff:ffff:ffff::/64
address = {{ slaprunner['ipv6'] }}/64
{% for partition_index in range(int(slaprunner['partition-amount'])) %}
[partition_{{ partition_index }}]
address = {{ slaprunner['ipv4'] }}/255.255.255.255 {{ slaprunner['ipv6'] }}/64
user = {{ partition_user }}
pathname = slappart{{ partition_index }}
network_interface =
{% endfor %}
software/slaprunner/template/slapos.cfg.in
View file @ eb1f843f
......@@ -22,6 +22,11 @@ pidfile_instance = {{slaprunner['pidfile-instance']}}
[slapformat]
partition_amount = {{ slaprunner['partition-amount'] }}
alter_user = false
alter_network = false
create_tab = false
create_tun = false
computer_xml = {{ slaprunner['etc_dir'] }}/slapos.xml
[slaprunner]
slapos = {{ slaprunner['slapos'] }}
......
software/slaprunner/template/supervisord.conf.in
View file @ eb1f843f
......@@ -44,6 +44,20 @@ stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[program:{{- supervisord['slapformat'] -}}]
command = {{ supervisord['slapformat-command'] }}
process_name = {{ supervisord['slapformat'] }}
numprocs = {{ supervisord['numprocs'] }}
autostart = {{ supervisord['autostart'] }}
exitcodes = {{ supervisord['exitcodes'] }}
startretries = {{ supervisord['slapformat-startretries'] }}
startsecs = {{ supervisord['startsecs'] }}
autorestart = {{ supervisord['autorestart'] }}
stdout_logfile = {{ supervisord['no_logfile'] }}
stderr_logfile = {{ supervisord['no_logfile'] }}
directory = {{ supervisord['directory'] }}
environment = PATH="{{- supervisord['path'] -}}"
[program:{{- supervisord['slapproxy'] -}}]
command = {{ supervisord['slapproxy-command'] }}
process_name = {{ supervisord['slapproxy'] }}
......
software/slaprunner/test/test.py
View file @ eb1f843f
......@@ -49,7 +49,6 @@ setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.join(os.path.dirname(__file__), '..',
'software%s.cfg' % ("-py3" if six.PY3 else ""))))
class SlaprunnerTestCase(SlapOSInstanceTestCase):
# Slaprunner uses unix sockets, so it needs short paths.
__partition_reference__ = 's'
......@@ -106,26 +105,6 @@ class TestWeb(SlaprunnerTestCase):
self.assertEqual(requests.codes.ok, hello.status_code)
self.assertIn('<b>Hello</b>', hello.text)
# git seems broken, these are 404 now...
@unittest.expectedFailure
def test_git_private(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = parameter_dict['git-private']
resp = requests.get(url, verify=False)
self.assertEqual(requests.codes.unauthorized, resp.status_code)
resp = requests.get(
url,
verify=False,
auth=(parameter_dict['init-user'], parameter_dict['init-password']))
self.assertEqual(requests.codes.ok, resp.status_code)
@unittest.expectedFailure
def test_git_public(self):
parameter_dict = self.computer_partition.getConnectionParameterDict()
url = parameter_dict['git-public']
resp = requests.get(url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
class TestSSH(SlaprunnerTestCase):
@classmethod
......@@ -275,6 +254,7 @@ class ServicesTestCase(SlaprunnerTestCase):
class TestInstanceResilient(SlaprunnerTestCase):
instance_max_retry = 20
@classmethod
def getInstanceSoftwareType(cls):
return 'resilient'
......@@ -287,8 +267,6 @@ class TestInstanceResilient(SlaprunnerTestCase):
'backend-url',
'feed-url-runner-1-pull',
'feed-url-runner-1-push',
'git-private-url',
'git-public-url',
'init-password',
'init-user',
'monitor-base-url',
......
software/turnserver/software.cfg
View file @ eb1f843f
......@@ -6,11 +6,10 @@ extends =
# ../../component/6tunnel/buildout.cfg
../../component/socat/buildout.cfg
../../stack/monitor/buildout.cfg
../../stack/slapos.cfg
buildout.hash.cfg
parts +=
parts =
slapos-cookbook
coturn
instance-cfg
......
stack/erp5/buildout.cfg
View file @ eb1f843f
......@@ -745,7 +745,6 @@ unidiff = 0.5.5
# Required by:
# deepdiff = 3.3.0
jsonpickle = 0.9.6
decorator = 4.3.0
responses = 0.10.6
......
stack/lamp/buildout.cfg
View file @ eb1f843f
......@@ -32,13 +32,11 @@ extends =
../../component/sqlite3/buildout.cfg
../../component/stunnel/buildout.cfg
../../component/zlib/buildout.cfg
../erp5/buildout.cfg
../logrotate/buildout.cfg
../resilient/buildout.cfg
../erp5/buildout.cfg
../../component/apache-php/buildout.cfg
../slapos.cfg
[template-download-base]
recipe = hexagonit.recipe.download
......@@ -47,10 +45,6 @@ download-only = true
url = ${:_profile_base_location_}/${:filename}
mode = 0644
[mariadb]
# Compile dir is for plugins, there's no plugin in LAMP
keep-compile-dir = false
[application]
recipe = hexagonit.recipe.download
ignore-existing = true
......@@ -146,5 +140,3 @@ eggs =
${lxml-python:egg}
${pycurl:egg}
slapos.toolbox
[versions]
stack/resilient/buildout.hash.cfg
View file @ eb1f843f
......@@ -46,7 +46,7 @@ md5sum = d21472f0e58f928fb827f2cbf22c4d4a
[resilient-web-takeover-cgi-script-download]
filename = resilient-web-takeover-cgi-script.py.in
md5sum = 7a0f1be7e9d05d0c68748322abe1f511
md5sum = 4137b7ac6b20a6cd897ad58f51a763b1
[template-wrapper]
filename = templates/wrapper.in
......
stack/resilient/resilient-web-takeover-cgi-script.py.in
View file @ eb1f843f
......@@ -48,7 +48,7 @@ def getLatestBackupDate():
if not db_keys:
result = False
else:
last_backup = db[db_keys[0]]
last_backup = float(db[db_keys[0]])
for callback in db_keys:
timestamp = float(db[callback])
if timestamp > last_backup:
......
stack/slapos.cfg
View file @ eb1f843f
......@@ -118,7 +118,7 @@ eggs =
[versions]
setuptools = 44.0.0
# Use SlapOS patched zc.buildout
zc.buildout = 2.7.1+slapos004
zc.buildout = 2.7.1+slapos006
# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
zc.recipe.egg = 2.0.3+slapos003
# Use own version of h.r.download to be able to open .xz and .lz archives
......@@ -126,7 +126,7 @@ hexagonit.recipe.download = 1.7.post4
Jinja2 = 2.9.5
MarkupSafe = 1.0
PyYAML = 3.12
PyYAML = 3.13
Werkzeug = 0.12
asn1crypto = 0.21.1
cffi = 1.9.1
......@@ -136,12 +136,12 @@ cmd2 = 0.7.0
collective.recipe.shelloutput = 0.1
collective.recipe.template = 2.0
cryptography = 2.3.1
decorator = 4.0.11
decorator = 4.3.0
idna = 2.2
inotify-simple = 1.1.1
itsdangerous = 0.24
lock-file = 2.0
lxml = 3.7.3
lxml = 4.4.3
meld3 = 1.0.2
netaddr = 0.7.19
pbr = 2.0.0
......
stack/lamp.cfg → stack/unstable/lamp.cfg
View file @ eb1f843f
......@@ -34,9 +34,6 @@ eggs =
${lxml-python:egg}
${pycrypto-python:egg}
[mariadb]
keep-compile-dir = false
[versions]
# Generated automatically by buildout-versions
async = 0.6.1
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7