Commit 42678c0d authored by Rafael Monnerat's avatar Rafael Monnerat

Update Release Candidate.

Conflicts:
	software/cloudooo/software.cfg
parents 84305f57 dc3fd2e2
From 6e2dd54b4988a40766db557906b1d544916a6859 Mon Sep 17 00:00:00 2001
From: Julien Muchembled <jm@nexedi.com>
Date: Fri, 13 Jan 2017 23:45:54 +0100
Subject: [PATCH] Fix deadlock with storages that "sync" on a new transaction
This backports a change from commit 227953b977a9e195c4ce9bbb9acd9c5ee60c333a.
NEO, as well as ZEO+server_sync (ERP5 backports this feature with a
monkey-patch), pings the server (primary master node in the case of NEO) on
new transactions. However, this round-trip is actually performed by the thread
that also does tasks requiring to lock the DB, like processing of invalidations.
Since transaction 1.6.1 (more precisely commit e581a120a6), IStorage.sync()
is called indirectly by DB.open() when a transaction has already begun,
and the DB must not be locked when this happens.
---
src/ZODB/DB.py | 9 +++------
src/ZODB/tests/testDB.py | 2 +-
2 files changed, 4 insertions(+), 7 deletions(-)
diff --git a/src/ZODB/DB.py b/src/ZODB/DB.py
index 95f1ab4..ffd10e9 100644
--- a/src/ZODB/DB.py
+++ b/src/ZODB/DB.py
@@ -753,19 +753,16 @@ def open(self, transaction_manager=None, at=None, before=None):
result = self.pool.pop()
assert result is not None
- # open the connection.
- result.open(transaction_manager)
-
# A good time to do some cache cleanup.
# (note we already have the lock)
self.pool.availableGC()
self.historical_pool.availableGC()
-
- return result
-
finally:
self._r()
+ result.open(transaction_manager)
+ return result
+
def connectionDebugInfo(self):
result = []
t = time.time()
diff --git a/src/ZODB/tests/testDB.py b/src/ZODB/tests/testDB.py
index 59b6ccf..e6286a4 100644
--- a/src/ZODB/tests/testDB.py
+++ b/src/ZODB/tests/testDB.py
@@ -151,7 +151,7 @@ def connectionDebugInfo():
>>> before
[None, '\x03zY\xd8\xc0m9\xdd', None]
>>> opened
- ['2008-12-04T20:40:44Z (1.40s)', '2008-12-04T20:40:45Z (0.30s)', None]
+ ['2008-12-04T20:40:44Z (1.30s)', '2008-12-04T20:40:46Z (0.10s)', None]
>>> infos
['test info (2)', ' (0)', ' (0)']
--
2.10.2.2.g19ca937.dirty
......@@ -22,13 +22,13 @@ parts =
[mariadb]
recipe = slapos.recipe.cmmi
version = 10.1.20
url = https://downloads.mariadb.org/f/mariadb-${:version}/source/mariadb-${:version}.tar.gz/from/http:/ftp.osuosl.org/pub/mariadb/?serve
md5sum = 623226918fd711e414ad240287284b5b
version = 10.1.21
url = http://ftp.osuosl.org/pub/mariadb/mariadb-10.1.21/source/mariadb-10.1.21.tar.gz
md5sum = ff8b96edeeb22425117309a249055310
location = ${buildout:parts-directory}/${:_buildout_section_name_}
patch-options = -p0
patches =
${:_profile_base_location_}/mariadb_10.1.8_create_system_tables__no_test.patch#d2f337e76b290071daf8985a79bda82a
${:_profile_base_location_}/mariadb_10.1.21_create_system_tables__no_test.patch#3c76aa9564a162f13aced7c0a3f783b3
configure-command = ${cmake:location}/bin/cmake
configure-options =
-DCMAKE_INSTALL_PREFIX=${:location}
......
......@@ -5,8 +5,8 @@ circumstances as it is checked first.
See http://bugs.debian.org/301741
and http://bugs.mysql.com/bug.php?id=6901
--- scripts/mysql_system_tables_data.sql.orig 2015-10-15 17:43:45.000000000 +0200
+++ scripts/mysql_system_tables_data.sql 2015-10-19 16:44:16.121393390 +0200
--- scripts/mysql_system_tables_data.sql.orig 2017-01-17 20:38:26.000000000 +0100
+++ scripts/mysql_system_tables_data.sql 2017-01-18 17:03:23.795082252 +0100
@@ -30,8 +30,6 @@
-- Fill "db" table with default grants for anyone to
-- access database 'test' and 'test_%' if "db" table didn't exist
......@@ -16,12 +16,13 @@ and http://bugs.mysql.com/bug.php?id=6901
INSERT INTO db SELECT * FROM tmp_db WHERE @had_db_table=0;
DROP TABLE tmp_db;
@@ -43,8 +41,6 @@
REPLACE INTO tmp_user SELECT @current_hostname,'root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0 FROM dual WHERE @current_hostname != 'localhost';
REPLACE INTO tmp_user VALUES ('127.0.0.1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N','',0);
REPLACE INTO tmp_user VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0);
-INSERT INTO tmp_user (host,user) VALUES ('localhost','');
-INSERT INTO tmp_user (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost';
INSERT INTO user SELECT * FROM tmp_user WHERE @had_user_table=0;
DROP TABLE tmp_user;
@@ -48,9 +46,6 @@
REPLACE INTO tmp_user_nopasswd VALUES ('::1','root','','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'','','N','N', '', 0);
-- More secure root account using unix sucket auth.
INSERT INTO tmp_user_socket VALUES ('localhost',IFNULL(@auth_root_socket, 'root'),'','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','Y','','','','',0,0,0,0,'unix_socket','','N', 'N','', 0);
--- Anonymous user with no privileges.
-INSERT INTO tmp_user_anonymous (host,user) VALUES ('localhost','');
-INSERT INTO tmp_user_anonymous (host,user) SELECT @current_hostname,'' FROM dual WHERE @current_hostname != 'localhost';
INSERT INTO user SELECT * FROM tmp_user_nopasswd WHERE @had_user_table=0 AND @skip_auth_root_nopasswd IS NULL;
INSERT INTO user SELECT * FROM tmp_user_socket WHERE @had_user_table=0 AND @auth_root_socket IS NOT NULL;
[buildout]
extends =
buildout.cfg
[mariarocks-source]
recipe = slapos.recipe.build:gitclone
git-executable = ${git:location}/bin/git
repository = https://github.com/MariaDB/server.git
branch = 10.2-mariarocks
[mariarocks]
<= mariadb
url =
md5sum =
patches =
path = ${mariarocks-source:location}
pre-configure =
${git:location}/bin/git submodule update --init
post-install =
make clean
......@@ -8,6 +8,17 @@ extends =
parts =
matplotlib
[matplotlibrc]
recipe = slapos.recipe.template:jinja2
location = ${buildout:parts-directory}/${:_buildout_section_name_}
template = ${:_profile_base_location_}/matplotlibrc.in
rendered = ${matplotlibrc:location}/matplotlibrc
mode = 0644
md5sum = 849f48f08bea21dd10d3825867cb1561
backend = agg
context =
key backend matplotlibrc:backend
[matplotlib-env]
<= numpy-env
PATH = ${pkgconfig:location}/bin:%(PATH)s
......@@ -22,4 +33,4 @@ setup-eggs = ${numpy:egg}
rpath =
${libpng:location}/lib
${freetype:location}/lib
need-matplotlibrc = ${matplotlibrc:location}
backend : {{ backend }}
\ No newline at end of file
......@@ -22,8 +22,8 @@ script =
wrapper_location = os.path.join("%(location)s", "x2t")
wrapper = open(wrapper_location, 'w')
wrapper.write('''#!${dash:location}/bin/dash
cd %(location)s
export LD_LIBRARY_PATH=%(location)s/lib:${curl:location}/lib
exec %(location)s/bin/x2t "$@"''')
wrapper.close()
os.chmod(wrapper_location, 0755)
os.chmod(location, 0750)
......@@ -13,7 +13,7 @@ extends =
[pkgconfig]
recipe = slapos.recipe.cmmi
url = http://pkgconfig.freedesktop.org/releases/pkg-config-0.29.tar.gz
url = https://pkgconfig.freedesktop.org/releases/pkg-config-0.29.tar.gz
md5sum = 77f27dce7ef88d0634d0d6f90e03a77f
location = ${buildout:parts-directory}/${:_buildout_section_name_}
# build pkg-config twice so that second configure can use pkg-config
......
......@@ -19,4 +19,4 @@ eggs =
# provided by SlapOS
initialization =
import os
os.environ['PATH'] = "${openvpn:location}/sbin:${babeld:location}/bin:${openssl:location}/bin:${python2.7:location}/bin:" + os.environ['PATH']
os.environ['PATH'] = "${openvpn:location}/sbin:${babeld:location}/bin:${openssl:location}/bin:${python2.7:location}/bin:" + os.environ.get('PATH', '/bin:/sbin:/usr/bin/:/usr/local/sbin')
......@@ -70,6 +70,12 @@ default_mimetype_entry_list = [
"application/emf * ooo",
"application/octet* * ooo",
"* application/vnd.oasis.opendocument* ooo",
"application/vnd.openxmlformats-officedocument.wordprocessingml.document application/x-asc-text x2t",
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet application/x-asc-spreadsheet x2t",
"application/vnd.openxmlformats-officedocument.presentationml.presentation application/x-asc-presentation x2t",
"application/x-asc-text application/vnd.openxmlformats-officedocument.wordprocessingml.document x2t",
"application/x-asc-spreadsheet application/vnd.openxmlformats-officedocument.spreadsheetml.sheet x2t",
"application/x-asc-presentation application/vnd.openxmlformats-officedocument.presentationml.presentation x2t",
]
class Recipe(GenericBaseRecipe):
......
......@@ -92,6 +92,9 @@ class Password(object):
- storage-path: plain-text persistent storage for password,
that can only be accessed by the user
(default: ${buildout:parts-directory}/${:_buildout_section_name_})
- create-once: boolean value which set if storage-path won't be modified
as soon the file is created with the password (not empty).
(default: True)
If storage-path is empty, the recipe does not save the password, which is
fine it is saved by other means, e.g. using the publish-early recipe.
......@@ -99,6 +102,8 @@ class Password(object):
def __init__(self, buildout, name, options):
options_get = options.get
self.create_once = options.get('create-once', 'True').lower() \
in GenericBaseRecipe.TRUE_VALUES
try:
self.storage_path = options['storage-path']
except KeyError:
......@@ -112,9 +117,9 @@ class Password(object):
except IOError as e:
if e.errno != errno.ENOENT:
raise
self.update = self.install
if not passwd:
passwd = self.generatePassword(int(options_get('bytes', '8')))
self.update = self.install
self.passwd = passwd
# Password must not go into .installed file, for 2 reasons:
# security of course but also to prevent buildout to always reinstall.
......@@ -126,16 +131,22 @@ class Password(object):
def install(self):
if self.storage_path:
try:
# The following 2 lines are just an optimization to avoid recreating
# the file with the same content.
if self.create_once and os.stat(self.storage_path).st_size:
return
os.unlink(self.storage_path)
except OSError as e:
if e.errno != errno.ENOENT:
raise
fd = os.open(self.storage_path,
os.O_CREAT | os.O_EXCL | os.O_WRONLY | os.O_TRUNC, 0600)
try:
os.write(fd, self.passwd)
finally:
os.close(fd)
if not self.create_once:
return self.storage_path
def update(self):
......
import os
import sys
import unittest
from tempfile import mkdtemp
from shutil import rmtree
from slapos.recipe import generic_cloudooo
class TestGenericCloudooo(unittest.TestCase):
def new_recipe(self, options):
buildout = {
'buildout': {
'bin-directory': '',
'find-links': '',
'allow-hosts': '',
'develop-eggs-directory': '',
'eggs-directory': '',
'python': 'testpython',
},
'testpython': {
'executable': sys.executable,
},
'slap-connection': {
'computer-id': '',
'partition-id': '',
'server-url': '',
'software-release-url': '',
}
}
return generic_cloudooo.Recipe(buildout=buildout, name='generic_cloudooo', options=options)
def setUp(self):
self.test_dir = mkdtemp()
def tearDown(self):
if os.path.exists(self.test_dir):
rmtree(self.test_dir)
def test_install(self):
# Basic check
config_file_path = os.path.join(self.test_dir, "test_install_configuration_file_etc_cloudooo-X.cfg")
recipe = self.new_recipe({
"ip": "test_install_ip",
"environment": "test_install=environment",
"mimetype_entry_addition": "text/install mimetype/entry addition",
"ooo-binary-path": "test_install_ooo_binary_path",
"ooo-paster": "test_install_ooo_paster",
"ooo-uno-path": "test_ooo_uno_path",
"port": "123",
"openoffice-port": "234",
"configuration-file": config_file_path,
"data-directory": os.path.join(self.test_dir, "test_install_data_directory_srv_cloudooo-X"),
"wrapper": os.path.join(self.test_dir, "test_install_wrapper_service_cloudooo-X"),
})
recipe.install()
data = open(config_file_path).read()
self.assertIn("[app:main]", data)
self.assertIn("[server:main]", data)
# Check if mimetype_registry is well ordered
self.assertIn("\n text/install mimetype/entry addition\n text/* * ooo\n", data)
# Check OnlyOffice entries
self.assertIn("\n".join([
"",
" application/vnd.openxmlformats-officedocument.presentationml.presentation application/x-asc-presentation x2t",
" application/vnd.openxmlformats-officedocument.spreadsheetml.sheet application/x-asc-spreadsheet x2t",
" application/vnd.openxmlformats-officedocument.wordprocessingml.document application/x-asc-text x2t",
"",
]), data)
self.assertIn("\n".join([
"",
" application/x-asc-presentation application/vnd.openxmlformats-officedocument.presentationml.presentation x2t",
" application/x-asc-spreadsheet application/vnd.openxmlformats-officedocument.spreadsheetml.sheet x2t",
" application/x-asc-text application/vnd.openxmlformats-officedocument.wordprocessingml.document x2t",
"",
]), data)
......@@ -18,14 +18,8 @@ timeout = 600
ssl-dict-parameter-name = ssl
#ssl-dict =
mimetype-entry-addition-parameter-name = mimetype-entry-addition
mimetype-entry-addition =
text/html application/pdf wkhtmltopdf
application/vnd.openxmlformats-officedocument.wordprocessingml.document application/x-asc-text x2t
application/vnd.openxmlformats-officedocument.spreadsheetml.sheet application/x-asc-spreadsheet x2t
application/vnd.openxmlformats-officedocument.presentationml.presentation application/x-asc-presentation x2t
application/x-asc-text application/vnd.openxmlformats-officedocument.wordprocessingml.document x2t
application/x-asc-spreadsheet application/vnd.openxmlformats-officedocument.spreadsheetml.sheet x2t
application/x-asc-presentation application/vnd.openxmlformats-officedocument.presentationml.presentation x2t
#mimetype-entry-addition =
# text/html application/pdf wkhtmltopdf
[cloudooo-repository]
branch = master
......
......@@ -10,3 +10,7 @@ branch = erp5-component
[erp5]
branch = erp5-component
[cloudooo-repository]
branch = master
revision =
......@@ -43,7 +43,7 @@ md5sum = d7d4a7e19d55bf14007819258bf42100
[erp5-kernel]
<= download-file-base
filename = ERP5kernel.py.jinja
md5sum = a0d13f4f5a9cd22a7b5d2c901aa9faae
md5sum = 24308ca010532863dbcf501f011f9846
[kernel-json]
<= download-file-base
......
......@@ -214,6 +214,8 @@ class ERP5Kernel(Kernel):
# Remove spaces and newlines from both ends of code
code = code.strip()
extra_data_list = []
if code.startswith('%'):
# No need to try-catch here as its already been taken that the code
# starts-with '%', so we'll get magic_name, no matter what be after '%'
......@@ -279,6 +281,8 @@ class ERP5Kernel(Kernel):
# Required in case the mime_type is anything other than 'text/plain'
mime_type = content['mime_type']
extra_data_list = content.get('extra_data_list', [])
# Display to frontend the error message for content status as 'error'
if content['status']=='error':
reply_content = {
......@@ -309,6 +313,9 @@ class ERP5Kernel(Kernel):
'metadata': {}}
self.send_response(self.iopub_socket, 'display_data', data)
for extra_data in extra_data_list:
self.send_response(self.iopub_socket, 'display_data', extra_data)
reply_content = {
'status': status,
# The base class increments the execution count
......
......@@ -13,7 +13,7 @@ SUMMARY_RE = re.compile(
r' (.*) (?P<duration>\d+(\.\d*)?|\.\d+)s', re.MULTILINE)
# NEO specific environment
TEMP_DIRECTORY = '{{directory.tmp}}/neo_tests'
TEMP_DIRECTORY = '{{directory.tmp}}'
NEO_DB_SOCKET = '{{my_cnf_parameters.socket}}'
RUN_NEO_TESTS_COMMAND = '{{ bin_directory }}/neotestrunner'
......@@ -82,13 +82,14 @@ def main():
if not test_result_line:
break
if os.path.exists(TEMP_DIRECTORY):
shutil.rmtree(TEMP_DIRECTORY)
os.mkdir(TEMP_DIRECTORY)
temp = os.path.join(TEMP_DIRECTORY, 'tests-' + test_result_line.name)
if os.path.exists(temp):
shutil.rmtree(temp)
os.mkdir(temp)
args = [RUN_NEO_TESTS_COMMAND, '-ufz']
command = ' '.join(args)
env = {'TEMP': TEMP_DIRECTORY,
env = {'TEMP': temp,
'NEO_TESTS_ADAPTER': test_result_line.name,
'NEO_TEST_ZODB_FUNCTIONAL': '1',
'NEO_DB_USER': 'root',
......
......@@ -47,6 +47,10 @@ eggs = neoppod[admin, ctl, master, storage-mysqldb]
ZODB
zope.testing
zodbtools
patch-binary = ${patch:location}/bin/patch
ZODB-patch-options = -p1
ZODB-patches =
${:_profile_base_location_}/../../component/egg-patch/ZODB4-pr136.patch
[slapos-deps-eggs]
recipe = zc.recipe.egg
......@@ -105,8 +109,8 @@ url = ${:_profile_base_location_}/my.cnf.in
md5sum = cfdd59d42ae540563a964ccaf8fec232
[versions]
slapos.recipe.template = 2.9
ZODB = 4.4.4
slapos.recipe.template = 2.10
ZODB = 4.4.4+SlapOSPatched001
BTrees = 4.3.1
persistent = 4.2.2
transaction = 1.7.0
......
[buildout]
extends = software.cfg
[neoppod]
ZODB-patches =
[versions]
ZODB = 5.1.1
transaction = 2.0.3
......@@ -26,7 +26,7 @@ md5sum = ee8401a4e7d82bf488a57e3399f9ce48
[runTestSuite.in]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/${:_buildout_section_name_}
md5sum = 1c8d903624310166629a173ecb8ad9f5
md5sum = 050593aef62fd4aa241d8ad378111c36
[runTestSuite_py]
recipe = zc.recipe.egg
......
......@@ -216,12 +216,14 @@ port = ${apache-conf:port}
{% do publish_dict.__setitem__('slave-amount', '${re6st-registry:slave-amount}') -%}
[publish]
recipe = slapos.cookbook:publish
monitor-setup-url = https://monitor.app.officejs.com/#page=settings_configurator&url=${publish:monitor-url}&username=${publish:monitor-user}&password=${publish:monitor-password}
{% for name, value in publish_dict.items() -%}
{{ name }} = {{ value }}
{% endfor -%}
[buildout]
extends =
{{ monitor2_template_rendered }}
{{ logrotate_cfg }}
parts =
......@@ -239,6 +241,7 @@ parts =
re6st-registry-promise
apache-registry-promise
monitor-base
# Complete parts with sections
{{ part_list | join('\n ') }}
......
......@@ -44,6 +44,7 @@ filename = instance-re6stnet.cfg
extensions = jinja2.ext.do
extra-context =
section parameter_dict dynamic-template-re6stnet-parameters
raw monitor2_template_rendered {{ monitor2_template_rendered }}
[switch-softwaretype]
recipe = slapos.cookbook:softwaretype
......
......@@ -12,7 +12,13 @@ extends =
../../component/python-cryptography/buildout.cfg
../../stack/slapos.cfg
parts =
# Monitoring stack (keep on bottom)
../../stack/monitor/buildout.cfg
extensions -=
buildout-versions
parts +=
slapos-cookbook-develop
slapos-cookbook
eggs
......@@ -65,7 +71,7 @@ context =
< = template-jinja2-base
filename = template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 1f39ec8d7d91c0ba2b4ad56f27ebac39
md5sum = 84387d5af13561447c1170704a2076a6
extra-context =
key apache_location apache:location
key dash_location dash:location
......@@ -76,12 +82,13 @@ extra-context =
key template_re6st_registry_conf template-re6st-registry-conf:target
key template_logrotate_base template-logrotate-base:rendered
key template_wrapper template-wrapper:target
key monitor2_template_rendered monitor2-template:rendered
raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
[template-re6stnet]
< = download-base
filename = instance-re6stnet.cfg.in
md5sum = 1f074a64d330e62213810a023ebce01e
md5sum = 050efe586eb78c0b4f0cc9ba33389675
[template-logrotate-base]
< = template-jinja2-base
......@@ -108,6 +115,9 @@ filename = wrapper.in
md5sum = 69e63cb58267335e21da772bd867657e
[versions]
setuptools = 28.8.0
zc.buildout = 2.5.2+slapos005
zc.recipe.egg = 2.0.3+slapos002
apache-libcloud = 0.17.0
ecdsa = 0.13
gitdb = 0.6.4
......
......@@ -77,4 +77,4 @@ revision = e8154cf06f01155f3f97d06954993f98bc1c9132
[versions]
msgpack-python = 0.4.8
wendelin.core = 0.8
wendelin.core = 0.9
......@@ -25,6 +25,7 @@ extends =
../../component/matplotlib/buildout.cfg
../../component/mesa/buildout.cfg
../../component/numpy/buildout.cfg
../../component/onlyoffice-x2t/buildout.cfg
../../component/pandas/buildout.cfg
../../component/percona-toolkit/buildout.cfg
../../component/patch/buildout.cfg
......@@ -39,6 +40,7 @@ extends =
../../component/tesseract/buildout.cfg
../../component/w3-validator/buildout.cfg
../../component/w3m/buildout.cfg
../../component/wkhtmltopdf/buildout.cfg
../../component/xorg/buildout.cfg
../../component/poppler/buildout.cfg
../../component/zabbix/buildout.cfg
......@@ -150,7 +152,7 @@ command = grep parts ${buildout:develop-eggs-directory}/slapos.cookbook.egg-link
[mariadb-resiliency-after-import-script]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/template/instance-mariadb-resiliency-after-import-script.sh.in
md5sum = b32d9ee1cb85f85d8d2f2b58f58459f1
md5sum = 79109ae720d5471cb2dd15e63fdf4dcf
mode = 755
[template-mariadb]
......@@ -177,17 +179,17 @@ md5sum = 763db0c4a94649296e74fe1f53c03940
[template-cloudooo]
<= download-base
filename = instance-cloudoo.cfg.in
md5sum = 5691b7f12d22bfb8a926dfcb592f12c8
md5sum = c553bbcdc5f80d893907a05acefc1356
[template-zope-conf]
<= download-base
filename = zope.conf.in
md5sum = 77ab4f229a92e02603028a0bd3772edd
[template-runzope-userhosts-preloaded]
[site-zcml]
<= download-base
filename = runzope_userhosts_preloaded.in
md5sum = bc6048b85b410693e60e5a77399dd1b7
filename = site.zcml
md5sum = d32417746fcf671d4e86a70379815039
[template-my-cnf]
<= download-base
......@@ -234,7 +236,7 @@ recipe = slapos.recipe.template:jinja2
# XXX: "template.cfg" is hardcoded in instanciation recipe
rendered = ${buildout:directory}/template.cfg
template = ${:_profile_base_location_}/instance.cfg.in
md5sum = 10d02ec69c875e6a55afe3bf79db7bba
md5sum = ec143441fac42ce6b3a183724298bc6b
mode = 640
context =
key mariadb_link_binary template-mariadb:link-binary
......@@ -262,6 +264,7 @@ context =
key glu_location glu:location
key gzip_location gzip:location
key haproxy_location haproxy:location
key imagemagick_location imagemagick:location
key instance_common_cfg instance-common:rendered
key jsl_location jsl:location
key jupyter_enable_default erp5-defaults:jupyter-enable-default
......@@ -283,11 +286,15 @@ context =
key logrotate_location logrotate:location
key mariadb_location mariadb:location
key mariadb_resiliency_after_import_script mariadb-resiliency-after-import-script:target
key matplotlibrc_location matplotlibrc:location
key mesa_location mesa:location
key onlyoffice_x2t_location onlyoffice-x2t:location
key openssl_location openssl:location
key pixman_location pixman:location
key poppler_location poppler:location
key postfix_location postfix:location
key root_common root-common:target
key site_zcml site-zcml:target
key sixtunnel_location 6tunnel:location
key slapos_core_version versions:slapos.core
key stunnel_location stunnel:location
......@@ -308,14 +315,15 @@ context =
key template_postfix_aliases template-postfix-aliases:target
key template_postfix_main_cf template-postfix-main-cf:target
key template_postfix_master_cf template-postfix-master-cf:target
key template_runzope_userhosts_preloaded template-runzope-userhosts-preloaded:target
key template_zeo template-zeo:target
key template_zope template-zope:target
key template_zope_conf template-zope-conf:target
key userhosts_location userhosts:location
key wget_location wget:location
key wkhtmltopdf_location wkhtmltopdf:location
key xdamage_location xdamage:location
key xfixes_location xfixes:location
key xserver_location xserver:location
key zlib_location zlib:location
key extra_path_list eggs:extra-paths
......@@ -336,12 +344,12 @@ md5sum = b5f49c90017684aa3389ef3f97ece509
[template-zeo]
<= download-base
filename = instance-zeo.cfg.in
md5sum = b0cb0ee97cddc79112a718e065806037
md5sum = 84aba4fa10d3087fbcc1d979df90fab9
[template-zope]
<= download-base
filename = instance-zope.cfg.in
md5sum = 83304f1940e0d7555bd678e2cea4c738
md5sum = e65672c423d603e5fa516ad811cc1fa9
link-binary =
${aspell:location}/bin/aspell
${dmtx-utils:location}/bin/dmtxwrite
......@@ -365,7 +373,7 @@ link-binary =
[template-balancer]
<= download-base
filename = instance-balancer.cfg.in
md5sum = ec9321514674c084e509ca070763b4a1
md5sum = 049d4153990374dc130403c04accaf13
[template-haproxy-cfg]
<= download-base
......@@ -565,6 +573,9 @@ eggs = ${neoppod:eggs}
# Needed for parsing .po files from our Localizer subset
polib
# Used by ERP5 Jupyter backend
ipykernel
# parameterizing the version of the generated python interpreter name by the
# python section version causes dependency between this egg section and the
# installation of python, which we don't want on an instance
......@@ -603,7 +614,7 @@ setup = ${erp5:location}
[cloudooo-repository]
branch =
revision = c4c753f2d9be7cda7084ed5fceda1fe6e01a73d8
revision = 00604a3a5a69082f7c607665cc6badd5104a32b9
[slapos-deps-eggs]
recipe = zc.recipe.egg
......
......@@ -43,7 +43,7 @@ ipv4 = {{ ipv4 }}
{% endif -%}
{% set haproxy_dict = {} -%}
{% set apache_dict = {} -%}
{% set next_port = slapparameter_dict['tcpv4-port'] -%}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% for family_name, parameter_id_list in sorted(
slapparameter_dict['zope-family-dict'].iteritems()) -%}
{% set zope_family_address_list = [] -%}
......@@ -55,14 +55,14 @@ ipv4 = {{ ipv4 }}
{% do has_webdav.append(None) %}
{% endif -%}
{% if use_ipv6 -%}
[{{ section('zope-tunnel-' ~ next_port) }}]
{% set current_port = next_port() -%}
[{{ section('zope-tunnel-' ~ current_port) }}]
< = zope-tunnel-base
base-name = {{ 'zeo-tunnel-' ~ next_port }}
ipv4-port = {{ next_port }}
base-name = {{ 'zeo-tunnel-' ~ current_port }}
ipv4-port = {{ current_port }}
ipv6-port = {{ zope_address.split(']:')[1] }}
ipv6 = {{ zope_address.split(']:')[0][1:] }}
{% set zope_effective_address = ipv4 ~ ":" ~ next_port -%}
{% set next_port = next_port + 1 -%}
{% set zope_effective_address = ipv4 ~ ":" ~ current_port -%}
{% else -%}
{% set zope_effective_address = zope_address -%}
{% endif -%}
......@@ -76,8 +76,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
# do a no-op getitem.
-#}
{% do zope_family_address_list[0][0] -%}
{% set haproxy_port = next_port -%}
{% set next_port = next_port + 1 -%}
{% set haproxy_port = next_port() -%}
{% do haproxy_dict.__setitem__(family_name, (haproxy_port, zope_family_address_list)) -%}
{% if has_webdav -%}
{% set internal_scheme = 'http' -%}{# mod_rewrite does not recognise webdav scheme -#}
......@@ -86,8 +85,7 @@ ipv6 = {{ zope_address.split(']:')[0][1:] }}
{% set internal_scheme = 'http' -%}
{% set external_scheme = 'https' -%}
{% endif -%}
{% do apache_dict.__setitem__(family_name, (next_port, external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ slapparameter_dict['backend-path'])) -%}
{% set next_port = next_port + 1 -%}
{% do apache_dict.__setitem__(family_name, (next_port(), external_scheme, internal_scheme ~ '://' ~ ipv4 ~ ':' ~ haproxy_port ~ slapparameter_dict['backend-path'])) -%}
{% endfor -%}
[haproxy-cfg-parameter-dict]
......@@ -209,7 +207,7 @@ crl = ${:ca-dir}/crl
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port }}
monitor-httpd-port = {{ next_port() }}
monitor-title = Balancer monitor
[buildout]
......
......@@ -10,6 +10,8 @@ parts +=
promise
promise-openoffice
{% if use_ipv6 %}promise-tunnel{% endif %}
xvfb-instance
wkhtmltopdf-on-xvfb
[publish]
recipe = slapos.cookbook:publish.serialised
......@@ -38,6 +40,7 @@ data-directory = ${directory:cloudooo-data}
environment =
LD_LIBRARY_PATH = {{ parameter_dict['cairo'] }}/lib:{{ parameter_dict['cups'] }}/lib:{{ parameter_dict['cups'] }}/lib64:{{ parameter_dict['dbus'] }}/lib:{{ parameter_dict['dbus-glib'] }}/lib:{{ parameter_dict['file'] }}/lib:{{ parameter_dict['fontconfig'] }}/lib:{{ parameter_dict['freetype'] }}/lib:{{ parameter_dict['glib'] }}/lib:{{ parameter_dict['glu'] }}/lib:{{ parameter_dict['libICE'] }}/lib:{{ parameter_dict['libSM'] }}/lib:{{ parameter_dict['libX11'] }}/lib:{{ parameter_dict['libXau'] }}/lib:{{ parameter_dict['libXdmcp'] }}/lib:{{ parameter_dict['libXext'] }}/lib:{{ parameter_dict['libXrender'] }}/lib:{{ parameter_dict['libexpat'] }}/lib:{{ parameter_dict['libffi'] }}/lib:{{ parameter_dict['libffi'] }}/lib64:{{ parameter_dict['libpng12'] }}/lib:{{ parameter_dict['libxcb'] }}/lib:{{ parameter_dict['mesa'] }}/lib:{{ parameter_dict['pixman'] }}/lib:{{ parameter_dict['xdamage'] }}/lib:{{ parameter_dict['xfixes'] }}/lib:{{ parameter_dict['zlib'] }}/lib
FONTCONFIG_FILE = ${fontconfig-instance:conf-path}
PATH = ${binary-link:target-directory}
# Binary information
# cloudooo specific configuration
......@@ -92,14 +95,55 @@ base-name = cloudooo-tunnel
# rest of parts are candidates for some generic stuff
[directory]
recipe = slapos.cookbook:mkdirectory
bin = ${buildout:directory}/bin
etc = ${buildout:directory}/etc
srv = ${buildout:directory}/srv
var = ${buildout:directory}/var
run = ${:var}/run
service = ${:etc}/run
promise = ${:etc}/promise
cloudooo-data = ${:srv}/cloudooo
font = ${:srv}/font
framebuffer = ${:srv}/framebuffer
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ tcpv4_port + 2 }}
monitor-title = Cloudooo monitor
[binary-link]
recipe = slapos.cookbook:symbolic.link
target-directory = ${directory:bin}
link-binary =
{{ parameter_dict['coreutils'] }}/bin/basename
{{ parameter_dict['coreutils'] }}/bin/cat
{{ parameter_dict['coreutils'] }}/bin/cp
{{ parameter_dict['coreutils'] }}/bin/ls
{{ parameter_dict['coreutils'] }}/bin/tr
{{ parameter_dict['coreutils'] }}/bin/uname
{{ parameter_dict['coreutils'] }}/bin/dirname
# wrapper recipe needs the head command
{{ parameter_dict['coreutils'] }}/bin/head
{{ parameter_dict['imagemagick'] }}/bin/convert
{{ parameter_dict['imagemagick'] }}/bin/identify
{{ parameter_dict['poppler'] }}/bin/pdfinfo
{{ parameter_dict['poppler'] }}/bin/pdftotext
{{ parameter_dict['poppler'] }}/bin/pdftohtml
{{ parameter_dict['onlyoffice-x2t'] }}/x2t
[xvfb-instance]
recipe = slapos.cookbook:xvfb
runner-path = ${directory:service}/xvfb
xvfb-path = {{ parameter_dict["xserver"] }}/bin/Xvfb
fbdir-path = ${directory:framebuffer}
tmp-path = ${directory:run}
shell-path = {{ parameter_dict["dash"] }}/bin/dash
[wkhtmltopdf-on-xvfb]
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/wkhtmltopdf
environment =
DISPLAY=:0
TMPDIR=${directory:run}
command-line = {{ parameter_dict['wkhtmltopdf'] }}/wkhtmltopdf --use-xserver
parameters-extra = true
{% set next_port = slapparameter_dict['tcpv4-port'] -%}
{% set next_port = itertools.count(slapparameter_dict['tcpv4-port']).next -%}
{% set ipv4 = (ipv4_set | list)[0] -%}
{% set backup_periodicity = slapparameter_dict.get('backup-periodicity', 'daily') -%}
{% set part_list = [] -%}
......@@ -27,12 +27,13 @@ ip = {{ ipv4 }}
{% for name, zodb in slapparameter_dict['zodb-dict'].iteritems() -%}
{% do zodb_dict.setdefault(zodb.get('family', 'default').lower(), []).append((name, zodb)) -%}
{% endfor -%}
{% set tidstorage_port = next_port + len(zodb_dict) -%}
{% set tidstorage_port = slapparameter_dict['tcpv4-port'] + len(zodb_dict) -%}
{% for family, zodb in zodb_dict.iteritems() -%}
{% set storage_list = [] -%}
{% set known_tid_storage_identifier_host = (ipv4, next_port), -%}
{% set current_port = next_port() -%}
{% set known_tid_storage_identifier_host = (ipv4, current_port), -%}
{% for name, zodb in zodb -%}
{% do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ next_port, 'storage': name}) %}
{% do storage_dict.__setitem__(name, {'server': ipv4 ~ ':' ~ current_port, 'storage': name}) %}
{% set path = zodb.get('path', '%(zodb)s/%(name)s.fs') % {'zodb': default_zodb_path, 'name': name} -%}
{% do storage_list.append((name, path)) -%}
{% set backup_directory = zodb.get('backup', '%(backup)s/%(name)s') % {'backup': default_backup_path, 'name': name} -%}
......@@ -48,7 +49,7 @@ ip = {{ ipv4 }}
[{{ zeo_section_name }}]
< = zeo-base
base-name = zeo-{{ family }}
port = {{ next_port }}
port = {{ current_port }}
storage = {{ dumps(storage_list) }}
[{{ section("logrotate-" ~ zeo_section_name) }}]
......@@ -63,7 +64,6 @@ hostname = {{ "${" ~ zeo_section_name ~ ":ip}" }}
port = {{ "${" ~ zeo_section_name ~ ":port}" }}
path = ${directory:promises}/zeo-{{ family }}
{% set next_port = next_port + 1 -%}
{% endfor -%}
{% if backup_periodicity == 'never' -%}
......@@ -185,10 +185,9 @@ backup-zodb = {{ default_backup_path }}
zodb = {{ default_zodb_path }}
tidstorage = {{ tidstorage_repozo_path }}
{% set next_port = next_port + 1 -%}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port }}
monitor-httpd-port = {{ next_port() }}
monitor-title = ZODB monitor
[buildout]
......
{% set use_ipv6 = slapparameter_dict.get('use-ipv6', False) -%}
{% set next_port = slapparameter_dict['port-base'] -%}
{% set next_port = itertools.count(slapparameter_dict['port-base']).next -%}
{% set site_id = slapparameter_dict['site-id'] -%}
{% set zodb_dict = slapparameter_dict['zodb-dict'] -%}
{% set node_id_base = slapparameter_dict['name'] -%}
......@@ -162,20 +162,6 @@ template = inline: {{ '
rendered = ${directory:etc}/hosts
context = key host_dict hosts-parameter:host-dict
[preload-userhosts-runzope-parameter]
runzope-binary = {{ bin_directory }}/runzope
userhosts = {{ parameter_dict['userhosts'] }}
shell-path = {{ parameter_dict['dash'] }}/bin/dash
hosts = ${hosts:rendered}
hostaliases = ${hostaliases:rendered}
[preload-userhosts-runzope]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/runzope_userhosts_preloaded
context = section parameter_dict preload-userhosts-runzope-parameter
template = {{ parameter_dict['runzope-userhosts-preloaded-template'] }}
mode = 755
{# Hack to deploy SSL certs via instance parameters -#}
{% for zodb in zodb_dict.itervalues() -%}
{% set storage_dict = zodb.setdefault('storage-dict', {}) -%}
......@@ -197,19 +183,32 @@ pem = {{dumps(storage_dict.pop(k))}}
{% endfor -%}
{# endhack -#}
[zope-base]
recipe = slapos.cookbook:generic.zope.zeo.client
inituser = ${directory:instance}/inituser
user = {{ dumps(slapparameter_dict['inituser-login']) }}
password = {{ dumps(slapparameter_dict['inituser-password']) }}
timezone = {{ dumps(slapparameter_dict['timezone']) }}
tmp-path = ${directory:tmp}
instancehome-path = ${directory:instance}
home-path = ${buildout:directory}
bin-path = ${directory:bin}:{{ parameter_dict['coreutils'] }}/bin
site-zcml = ${directory:instance-etc}/site.zcml
runzope-binary = ${preload-userhosts-runzope:rendered}
bt5-repository =
[runzope-base]
recipe = slapos.cookbook:wrapper
environment =
HOSTALIASES=${hostaliases:rendered}
HOSTS=${hosts:rendered}
TMP=${directory:tmp}
TMPDIR=${directory:tmp}
HOME=${buildout:directory}
PATH=${directory:bin}:{{ parameter_dict['coreutils'] }}/bin
TZ={{ slapparameter_dict['timezone'] }}
INSTANCE_HOME=${directory:instance}
MATPLOTLIBRC={{ parameter_dict['matplotlibrc'] }}
parameters-extra = true
command-line = '{{ parameter_dict['userhosts'] }}' '{{ bin_directory }}/runzope' -C '${:configuration-file}'
[{{ section('zcml') }}]
recipe = slapos.cookbook:copyfilelist
target-directory = ${directory:instance-etc}
file-list = {{ parameter_dict['site-zcml'] }}
[{{ section('zope-inituser') }}]
recipe = slapos.recipe.template:jinja2
rendered = ${directory:instance}/inituser
template = inline:{{ slapparameter_dict['inituser-login'] }}:{SHA}{{ hashlib.sha1(slapparameter_dict['inituser-password']).digest().encode('base64').rstrip() }}
mode = 600
once = ${:rendered}_done
[zope-conf-parameter-base]
ip = {{ ipv4 }}
......@@ -309,8 +308,8 @@ context =
section parameter_dict {{ conf_parameter_name }}
[{{ section(name) }}]
< = zope-base
wrapper = ${directory:service-on-watch}/{{ name }}
< = runzope-base
wrapper-path = ${directory:service-on-watch}/{{ name }}
configuration-file = {{ '${' ~ conf_name ~ ':rendered}' }}
[{{ section("promise-" ~ name) }}]
......@@ -359,11 +358,10 @@ post = {{ bin_directory }}/slapos-kill --pidfile {{ '${' ~ conf_parameter_name ~
{% for i in range(slapparameter_dict['instance-count']) -%}
{{ zope(
i,
next_port,
next_port(),
slapparameter_dict['longrequest-logger-timeout'],
slapparameter_dict['longrequest-logger-interval'],
) }}
{% set next_port = next_port + 1 -%}
{% endfor -%}
[publish]
......@@ -390,7 +388,7 @@ bt5-repository-url = {{ dumps(slapparameter_dict['bt5-repository-url']) }}
[monitor-instance-parameter]
monitor-httpd-ipv6 = {{ (ipv6_set | list)[0] }}
monitor-httpd-port = {{ next_port }}
monitor-httpd-port = {{ next_port() }}
monitor-title = Zope monitor
[buildout]
......
......@@ -7,6 +7,7 @@ context +=
[dynamic-template-cloudooo-parameters]
cairo = {{ cairo_location }}
coreutils = {{ dumps(coreutils_location) }}
cups = {{ cups_location }}
dbus = {{ dbus_location }}
dbus-glib = {{ dbus_glib_location }}
......@@ -15,6 +16,7 @@ fontconfig = {{ fontconfig_location }}
freetype = {{ freetype_location }}
glib = {{ glib_location }}
glu = {{ glu_location }}
imagemagick = {{ dumps(imagemagick_location) }}
libICE = {{ libICE_location }}
libSM = {{ libSM_location }}
libX11 = {{ libX11_location }}
......@@ -27,9 +29,13 @@ libffi = {{ libffi_location }}
libpng12 = {{ libpng12_location }}
libxcb = {{ libxcb_location }}
mesa = {{ mesa_location }}
onlyoffice-x2t = {{ dumps(onlyoffice_x2t_location) }}
pixman = {{ pixman_location }}
poppler = {{ dumps(poppler_location) }}
wkhtmltopdf = {{ dumps(wkhtmltopdf_location) }}
xdamage = {{ xdamage_location }}
xfixes = {{ xfixes_location }}
xserver = {{ dumps(xserver_location) }}
zlib = {{ zlib_location }}
libreoffice-bin = {{ libreoffice_bin_location }}
fonts = {{ fonts_location }}
......@@ -98,6 +104,7 @@ template = {{ template_balancer }}
filename = instance-balancer.cfg
extra-context =
section parameter_dict dynamic-template-balancer-parameters
import itertools itertools
[dynamic-template-zeo-parameters]
buildout-bin-directory = {{ buildout_bin_directory }}
......@@ -111,6 +118,7 @@ extra-context =
key buildout_directory buildout:directory
section parameter_dict dynamic-template-zeo-parameters
import json_module json
import itertools itertools
[dynamic-template-zope-parameters]
bin-directory = {{ bin_directory }}
......@@ -122,9 +130,10 @@ dash = {{ dash_location }}
jsl = {{ jsl_location }}
link-binary = {{ dumps(zope_link_binary) }}
userhosts = {{ userhosts_location }}
runzope-userhosts-preloaded-template = {{ template_runzope_userhosts_preloaded }}
site-zcml = {{ site_zcml }}
template-monitor = {{ dumps(template_monitor) }}
extra-path-list = {{ dumps(extra_path_list) }}
matplotlibrc = {{ matplotlibrc_location }}
[dynamic-template-zope]
<= jinja2-template-base
......@@ -134,6 +143,8 @@ extra-context =
key buildout_directory buildout:directory
section parameter_dict dynamic-template-zope-parameters
import urlparse urlparse
import hashlib hashlib
import itertools itertools
[dynamic-template-kumofs-parameters]
dash-location = {{ dash_location }}
......
#!{{ parameter_dict['shell-path'] }}
HOSTALIASES='{{ parameter_dict['hostaliases'] }}' HOSTS='{{ parameter_dict['hosts'] }}' exec '{{ parameter_dict['userhosts'] }}' '{{ parameter_dict['runzope-binary'] }}' "$@"
<configure
xmlns="http://namespaces.zope.org/zope"
xmlns:meta="http://namespaces.zope.org/meta"
xmlns:five="http://namespaces.zope.org/five">
<include package="Products.Five" />
<meta:redefinePermission from="zope2.Public" to="zope.Public" />
<!-- Load the meta -->
<include files="package-includes/*-meta.zcml" />
<five:loadProducts file="meta.zcml"/>
<!-- Load the configuration -->
<include files="package-includes/*-configure.zcml" />
<five:loadProducts />
<!-- Load the configuration overrides-->
<includeOverrides files="package-includes/*-overrides.zcml" />
<five:loadProductsOverrides />
<securityPolicy
component="Products.Five.security.FiveSecurityPolicy" />
</configure>
......@@ -32,11 +32,16 @@ fi
echo "Deleting existing database..."
rm -r $mariadb_data_directory/* >/dev/null 2>&1 || true
echo "Adapting binlog database to new paths..."
new_binlog_directory="$(dirname $binlog_path)"
binlog_index_file="$new_binlog_directory/binlog.index"
old_binlog_directory="$(dirname $(head -n 1 $binlog_index_file))"
sed -e "s|$old_binlog_directory|$new_binlog_directory|g" $binlog_index_file > $binlog_index_file
# $binlog_path can be empty if incremental_backup_retention_days <= -1
if [ -n "$binlog_path" ]; then
new_binlog_directory="$(dirname $binlog_path)"
binlog_index_file="$new_binlog_directory/binlog.index"
if [ -e "$binlog_index_file" ]; then
echo "Adapting binlog database to new paths..."
old_binlog_directory="$(dirname $(head -n 1 $binlog_index_file))"
sed -e "s|$old_binlog_directory|$new_binlog_directory|g" $binlog_index_file > $binlog_index_file
fi
fi
echo "Starting mariadb..."
# XXX hardcoded
......
......@@ -137,7 +137,7 @@ setuptools = 19.6.2
six = 1.10.0
slapos.cookbook = 1.0.45
slapos.core = 1.3.18
slapos.extension.strip = 0.1
slapos.extension.strip = 0.2
slapos.libnetworkcache = 0.14.5
slapos.recipe.build = 0.28
slapos.recipe.cmmi = 0.2
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment