Commit 63055263 authored by Rafael Monnerat's avatar Rafael Monnerat

Update Release Candidate

parents da630113 865bc5d4
...@@ -70,7 +70,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce ...@@ -70,7 +70,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
[template-trafficserver-records-config] [template-trafficserver-records-config]
_update_hash_filename_ = templates/trafficserver/records.config.jinja2 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
md5sum = f3f31188de56bb35383335b3219537f4 md5sum = 56c7e9f5cc92f8c519f48496010e2078
[template-trafficserver-storage-config] [template-trafficserver-storage-config]
_update_hash_filename_ = templates/trafficserver/storage.config.jinja2 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2
......
...@@ -27,6 +27,7 @@ CONFIG proxy.config.http.cache.open_write_fail_action INT 2 ...@@ -27,6 +27,7 @@ CONFIG proxy.config.http.cache.open_write_fail_action INT 2
CONFIG proxy.config.body_factory.template_sets_dir STRING {{ ats_configuration['templates-dir'] }} CONFIG proxy.config.body_factory.template_sets_dir STRING {{ ats_configuration['templates-dir'] }}
# Support stale-if-error by returning cached content on backend 5xx or unavailability # Support stale-if-error by returning cached content on backend 5xx or unavailability
CONFIG proxy.config.http.negative_revalidating_enabled INT 1 CONFIG proxy.config.http.negative_revalidating_enabled INT 1
CONFIG proxy.config.http.negative_revalidating_lifetime INT 86400
############################################################################## ##############################################################################
# Proxy users variables. Docs: # Proxy users variables. Docs:
# https://docs.trafficserver.apache.org/records.config#proxy-user-variables # https://docs.trafficserver.apache.org/records.config#proxy-user-variables
......
...@@ -66,6 +66,7 @@ from cryptography.x509.oid import NameOID ...@@ -66,6 +66,7 @@ from cryptography.x509.oid import NameOID
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.testing.utils import findFreeTCPPort from slapos.testing.utils import findFreeTCPPort
from slapos.testing.utils import getPromisePluginParameterDict
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
...@@ -243,44 +244,6 @@ def isHTTP2(domain, ip): ...@@ -243,44 +244,6 @@ def isHTTP2(domain, ip):
return 'Using HTTP2, server supports multi-use' in err return 'Using HTTP2, server supports multi-use' in err
def getPluginParameterDict(software_path, filepath):
"""Load the slapos monitor plugin and returns the configuration used by this plugin.
This allow to check that monitoring plugin are using a proper config.
"""
# This is implemented by creating a wrapper script that loads the plugin wrapper
# script and returns its `extra_config_dict`. This might have to be adjusted if
# internals of slapos promise plugins change.
bin_file = os.path.join(software_path, 'bin', 'test-plugin-promise')
monitor_python_with_eggs = os.path.join(software_path, 'bin', 'monitor-pythonwitheggs')
if not os.path.exists(monitor_python_with_eggs):
raise ValueError("Monitoring stack's python does not exist at %s" % monitor_python_with_eggs)
with open(bin_file, 'w') as f:
f.write("""#!%s
import os
import importlib
import sys
import json
filepath = sys.argv[1]
sys.path[0:0] = [os.path.dirname(filepath)]
filename = os.path.basename(filepath)
module = importlib.import_module(os.path.splitext(filename)[0])
print json.dumps(module.extra_config_dict)
""" % monitor_python_with_eggs)
os.chmod(bin_file, 0o755)
result = subprocess_output([bin_file, filepath]).strip()
try:
return json.loads(result)
except ValueError, e:
raise ValueError("%s\nResult was: %s" % (e, result))
class TestDataMixin(object): class TestDataMixin(object):
def getTrimmedProcessInfo(self): def getTrimmedProcessInfo(self):
return '\n'.join(sorted([ return '\n'.join(sorted([
...@@ -3569,7 +3532,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -3569,7 +3532,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
'check-_monitor-ipv6-test-ipv6-packet-list-test.py'))[0] 'check-_monitor-ipv6-test-ipv6-packet-list-test.py'))[0]
# get promise module and check that parameters are ok # get promise module and check that parameters are ok
self.assertEqual( self.assertEqual(
getPluginParameterDict(self.software_path, monitor_file), getPromisePluginParameterDict(monitor_file),
{ {
'frequency': '720', 'frequency': '720',
'address': 'monitor-ipv6-test' 'address': 'monitor-ipv6-test'
...@@ -3606,7 +3569,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin): ...@@ -3606,7 +3569,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
'check-_monitor-ipv4-test-ipv4-packet-list-test.py'))[0] 'check-_monitor-ipv4-test-ipv4-packet-list-test.py'))[0]
# get promise module and check that parameters are ok # get promise module and check that parameters are ok
self.assertEqual( self.assertEqual(
getPluginParameterDict(self.software_path, monitor_file), getPromisePluginParameterDict(monitor_file),
{ {
'frequency': '720', 'frequency': '720',
'ipv4': 'true', 'ipv4': 'true',
...@@ -4702,8 +4665,7 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase, ...@@ -4702,8 +4665,7 @@ class TestRe6stVerificationUrlDefaultSlave(SlaveHttpFrontendTestCase,
re6st_connectivity_promise_file = re6st_connectivity_promise_list[0] re6st_connectivity_promise_file = re6st_connectivity_promise_list[0]
self.assertEqual( self.assertEqual(
getPluginParameterDict( getPromisePluginParameterDict(re6st_connectivity_promise_file),
self.software_path, re6st_connectivity_promise_file),
{ {
'url': 'http://[2001:67c:1254:4::1]/index.html', 'url': 'http://[2001:67c:1254:4::1]/index.html',
} }
...@@ -4757,8 +4719,7 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase, ...@@ -4757,8 +4719,7 @@ class TestRe6stVerificationUrlSlave(SlaveHttpFrontendTestCase,
re6st_connectivity_promise_file = re6st_connectivity_promise_list[0] re6st_connectivity_promise_file = re6st_connectivity_promise_list[0]
self.assertEqual( self.assertEqual(
getPluginParameterDict( getPromisePluginParameterDict(re6st_connectivity_promise_file),
self.software_path, re6st_connectivity_promise_file),
{ {
'url': 'some-re6st-verification-url', 'url': 'some-re6st-verification-url',
} }
...@@ -6462,7 +6423,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase): ...@@ -6462,7 +6423,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
# get promise module and check that parameters are ok # get promise module and check that parameters are ok
self.assertEqual( self.assertEqual(
getPluginParameterDict(self.software_path, monitor_file), getPromisePluginParameterDict(monitor_file),
{ {
'frequency': '720', 'frequency': '720',
'ipv4': 'true', 'ipv4': 'true',
...@@ -6506,7 +6467,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase): ...@@ -6506,7 +6467,7 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
'check-_monitor-ipv6-test-unsafe-ipv6-packet-list-test.py'))[0] 'check-_monitor-ipv6-test-unsafe-ipv6-packet-list-test.py'))[0]
# get promise module and check that parameters are ok # get promise module and check that parameters are ok
self.assertEqual( self.assertEqual(
getPluginParameterDict(self.software_path, monitor_file), getPromisePluginParameterDict(monitor_file),
{ {
'frequency': '720', 'frequency': '720',
'address': '${section:option}\nafternewline ipv6' 'address': '${section:option}\nafternewline ipv6'
......
...@@ -48,6 +48,7 @@ setup(name=name, ...@@ -48,6 +48,7 @@ setup(name=name,
'psutil', 'psutil',
'requests', 'requests',
'mysqlclient', 'mysqlclient',
'backports.lzma',
'cryptography', 'cryptography',
'pyOpenSSL', 'pyOpenSSL',
], ],
......
...@@ -47,20 +47,17 @@ class TestPublishedURLIsReachableMixin(object): ...@@ -47,20 +47,17 @@ class TestPublishedURLIsReachableMixin(object):
# What happens is that instanciation just create the services, but does not # What happens is that instanciation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is # wait for ERP5 to be initialized. When this test run ERP5 instance is
# instanciated, but zope is still busy creating the site and haproxy replies # instanciated, but zope is still busy creating the site and haproxy replies
# with 503 Service Unavailable, sometimes the first request is 404, so we # with 503 Service Unavailable when zope is not started yet, with 404 when
# retry in a loop. # erp5 site is not created, with 500 when mysql is not yet reachable, so we
# If we can move the "create site" in slapos node instance, then this retry loop # retry in a loop until we get a succesful response.
# would not be necessary.
for i in range(1, 60): for i in range(1, 60):
r = requests.get(url, verify=False) # XXX can we get CA from caucase already ? r = requests.get(url, verify=False) # XXX can we get CA from caucase already ?
if r.status_code in (requests.codes.service_unavailable, if r.status_code != requests.codes.ok:
requests.codes.not_found):
delay = i * 2 delay = i * 2
self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay) self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
time.sleep(delay) time.sleep(delay)
continue continue
if r.status_code != requests.codes.ok: r.raise_for_status()
r.raise_for_status()
break break
self.assertIn("ERP5", r.text) self.assertIn("ERP5", r.text)
......
...@@ -31,12 +31,19 @@ import json ...@@ -31,12 +31,19 @@ import json
import glob import glob
import urlparse import urlparse
import socket import socket
import sys
import time import time
import contextlib import contextlib
import datetime import datetime
import subprocess
import gzip
from backports import lzma
import MySQLdb import MySQLdb
from slapos.testing.utils import CrontabMixin
from slapos.testing.utils import getPromisePluginParameterDict
from . import ERP5InstanceTestCase from . import ERP5InstanceTestCase
from . import setUpModule from . import setUpModule
setUpModule # pyflakes setUpModule # pyflakes
...@@ -56,8 +63,8 @@ class MariaDBTestCase(ERP5InstanceTestCase): ...@@ -56,8 +63,8 @@ class MariaDBTestCase(ERP5InstanceTestCase):
return { return {
'tcpv4-port': 3306, 'tcpv4-port': 3306,
'max-connection-count': 5, 'max-connection-count': 5,
'max-slowqueries-threshold': 5, 'max-slowqueries-threshold': 1,
'slowest-query-threshold': 10, 'slowest-query-threshold': 0.1,
# XXX what is this ? should probably not be needed here # XXX what is this ? should probably not be needed here
'name': cls.__name__, 'name': cls.__name__,
'monitor-passwd': 'secret', 'monitor-passwd': 'secret',
...@@ -86,6 +93,88 @@ class MariaDBTestCase(ERP5InstanceTestCase): ...@@ -86,6 +93,88 @@ class MariaDBTestCase(ERP5InstanceTestCase):
) )
class TestCrontabs(MariaDBTestCase, CrontabMixin):
def test_full_backup(self):
self._executeCrontabAtDate('mariadb-backup', '2050-01-01')
with gzip.open(
os.path.join(
self.computer_partition_root_path,
'srv',
'backup',
'mariadb-full',
'20500101000000.sql.gz',
),
'r') as dump:
self.assertIn('CREATE TABLE', dump.read())
def test_logrotate_and_slow_query_digest(self):
# slow query digest needs to run after logrotate, since it operates on the rotated
# file, so this tests both logrotate and slow query digest.
# run logrotate a first time so that it create state files
self._executeCrontabAtDate('logrotate', '2000-01-01')
# make two slow queries
cnx = self.getDatabaseConnection()
with contextlib.closing(cnx):
cnx.query("SELECT SLEEP(1.1)")
cnx.store_result()
cnx.query("SELECT SLEEP(1.2)")
# slow query crontab depends on crontab for log rotation
# to be executed first.
self._executeCrontabAtDate('logrotate', '2050-01-01')
# this logrotate leaves the log for the day as non compressed
rotated_log_file = os.path.join(
self.computer_partition_root_path,
'srv',
'backup',
'logrotate',
'mariadb_slowquery.log-20500101',
)
self.assertTrue(os.path.exists(rotated_log_file))
# then crontab to generate slow query report is executed
self._executeCrontabAtDate('generate-mariadb-slow-query-report', '2050-01-01')
# and it creates a report for the day
slow_query_report = os.path.join(
self.computer_partition_root_path,
'srv',
'monitor',
'private',
'slowquery_digest',
'slowquery_digest.txt-2050-01-01.xz',
)
with lzma.open(slow_query_report, 'r') as f:
# this is the hash for our "select sleep(n)" slow query
self.assertIn("ID 0xF9A57DD5A41825CA", f.read())
# on next day execution of logrotate, log files are compressed
self._executeCrontabAtDate('logrotate', '2050-01-02')
self.assertTrue(os.path.exists(rotated_log_file + '.xz'))
self.assertFalse(os.path.exists(rotated_log_file))
# there's a promise checking that the threshold is not exceeded
# and it reports a problem since we set a threshold of 1 slow query
check_slow_query_promise_plugin = getPromisePluginParameterDict(
os.path.join(
self.computer_partition_root_path,
'etc',
'plugin',
'check-slow-query-pt-digest-result.py',
))
with self.assertRaises(subprocess.CalledProcessError) as error_context:
subprocess.check_output('faketime 2050-01-01 %s' % check_slow_query_promise_plugin['command'], shell=True)
self.assertEqual(
error_context.exception.output,
"""\
Threshold is lower than expected:
Expected total queries : 1.0 and current is: 2
Expected slowest query : 0.1 and current is: 1
""")
class TestMariaDB(MariaDBTestCase): class TestMariaDB(MariaDBTestCase):
def test_utf8_collation(self): def test_utf8_collation(self):
cnx = self.getDatabaseConnection() cnx = self.getDatabaseConnection()
......
...@@ -11,7 +11,7 @@ parts = ...@@ -11,7 +11,7 @@ parts =
recipe = slapos.recipe.template recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance.cfg url = ${:_profile_base_location_}/instance.cfg
output = ${buildout:directory}/template.cfg output = ${buildout:directory}/template.cfg
md5sum = efd3b712a2294207f265a9c45648d5cf md5sum = 9fccb9600a085691ab0f9a20cd615c01
mode = 0644 mode = 0644
[versions] [versions]
......
...@@ -56,8 +56,12 @@ config-cpu-usage-ratio = 1 ...@@ -56,8 +56,12 @@ config-cpu-usage-ratio = 1
config-software-root = ${buildout:directory}/../../soft config-software-root = ${buildout:directory}/../../soft
config-buildout-shared-folder = ${buildout:directory}/../../shared config-buildout-shared-folder = ${buildout:directory}/../../shared
config-no-ipv4-frontend = true config-no-ipv4-frontend = true
# XXX hardcoded
#config-frontend-domain = google.com # Use same repository / branch for the software installed in slaprunner
# than the tested slaprunner itself.
config-slapos-repository = {{ slapos_repository_url }}
config-slapos-reference = {{ slapos_repository_branch }}
# XXX Hack to deploy Root Instance on the same computer as the type-test Instance # XXX Hack to deploy Root Instance on the same computer as the type-test Instance
sla-computer_guid = ${slap-connection:computer-id} sla-computer_guid = ${slap-connection:computer-id}
return = backend-url return = backend-url
......
...@@ -7,7 +7,16 @@ parts += template-erp5testnode ...@@ -7,7 +7,16 @@ parts += template-erp5testnode
[template-resilient-test] [template-resilient-test]
filename = instance-resilient-test.cfg.jinja2 filename = instance-resilient-test.cfg.jinja2
md5sum = cf34be404bdd93a72df9c593f8762cb2 md5sum = be4adbc196f07f5e39ca78401dfa53ec
# We have to use an extra level of indentation here because this is substituted
# during software buildout to generate instance buildout, but the
# slapos.recipe.template recipe doing the substitution does string replacements
# without knowledge of the buildout syntax, so we want the second line to be
# indented in the final generated instance buildout.
extra-context =
raw slapos_repository_url ${slapos.cookbook-repository:repository}
raw slapos_repository_branch ${slapos.cookbook-repository:branch}
[exporter-default-configuration] [exporter-default-configuration]
# Define shorter interaction to speed up tests # Define shorter interaction to speed up tests
......
...@@ -744,9 +744,9 @@ class TestCpuMemMaxDynamic(InstanceTestCase): ...@@ -744,9 +744,9 @@ class TestCpuMemMaxDynamic(InstanceTestCase):
def test(self): def test(self):
with open(os.path.join( with open(os.path.join(
self.computer_partition_root_path, 'bin', 'kvm_raw'), 'rb') as fh: self.computer_partition_root_path, 'bin', 'kvm_raw'), 'r') as fh:
kvm_raw = fh.read() kvm_raw = fh.read()
self.assertTrue('smp_count = 2' in kvm_raw) self.assertIn('smp_count = 2', kvm_raw)
self.assertTrue('smp_max_count = 3' in kvm_raw) self.assertIn('smp_max_count = 3', kvm_raw)
self.assertTrue('ram_size = 2048' in kvm_raw) self.assertIn('ram_size = 2048', kvm_raw)
self.assertTrue("ram_max_size = '2560'" in kvm_raw) self.assertIn("ram_max_size = '2560'", kvm_raw)
...@@ -34,7 +34,7 @@ md5sum = 74e0361f3ec3424c905acc4cd55fd8bf ...@@ -34,7 +34,7 @@ md5sum = 74e0361f3ec3424c905acc4cd55fd8bf
[template-neo-my-cnf] [template-neo-my-cnf]
filename = my.cnf.in filename = my.cnf.in
md5sum = 9f6f8f2b5f4cb0d97d50ffc1d3837e2f md5sum = a8939dcf440126a30db661b5a0a1bfb7
[template-neo] [template-neo]
filename = instance.cfg.in filename = instance.cfg.in
......
...@@ -9,6 +9,8 @@ ...@@ -9,6 +9,8 @@
{% if not engine or engine == 'RocksDB' -%} {% if not engine or engine == 'RocksDB' -%}
{% do plugins.append('ha_rocksdb') -%} {% do plugins.append('ha_rocksdb') -%}
{% do extra_dict.setdefault('rocksdb_keep_log_file_num', '64') -%}
{% do extra_dict.setdefault('rocksdb_max_log_file_size', '16M') -%}
{% endif -%} {% endif -%}
{% if not engine or engine == 'TokuDB' -%} {% if not engine or engine == 'TokuDB' -%}
......
...@@ -47,20 +47,17 @@ class TestPublishedURLIsReachableMixin(object): ...@@ -47,20 +47,17 @@ class TestPublishedURLIsReachableMixin(object):
# What happens is that instanciation just create the services, but does not # What happens is that instanciation just create the services, but does not
# wait for ERP5 to be initialized. When this test run ERP5 instance is # wait for ERP5 to be initialized. When this test run ERP5 instance is
# instanciated, but zope is still busy creating the site and haproxy replies # instanciated, but zope is still busy creating the site and haproxy replies
# with 503 Service Unavailable, sometimes the first request is 404, so we # with 503 Service Unavailable when zope is not started yet, with 404 when
# retry in a loop. # erp5 site is not created, with 500 when mysql is not yet reachable, so we
# If we can move the "create site" in slapos node instance, then this retry loop # retry in a loop until we get a succesful response.
# would not be necessary.
for i in range(1, 60): for i in range(1, 60):
r = requests.get(url, verify=False) # XXX can we get CA from caucase already ? r = requests.get(url, verify=False) # XXX can we get CA from caucase already ?
if r.status_code in (requests.codes.service_unavailable, if r.status_code != requests.codes.ok:
requests.codes.not_found):
delay = i * 2 delay = i * 2
self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay) self.logger.warn("ERP5 was not available, sleeping for %ds and retrying", delay)
time.sleep(delay) time.sleep(delay)
continue continue
if r.status_code != requests.codes.ok: r.raise_for_status()
r.raise_for_status()
break break
self.assertIn("ERP5", r.text) self.assertIn("ERP5", r.text)
......
...@@ -15,4 +15,4 @@ ...@@ -15,4 +15,4 @@
[template] [template]
filename = instance.cfg filename = instance.cfg
md5sum = 25a4d7e438402d992edadf9339faf557 md5sum = 14d2f49d20670e44c2a162bcec9e0a8e
...@@ -44,7 +44,7 @@ environment = ...@@ -44,7 +44,7 @@ environment =
SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR} SLAPOS_TEST_WORKING_DIR=${slapos-test-runner-environment:SLAPOS_TEST_WORKING_DIR}
[slapos-test-runner-environment] [slapos-test-runner-environment]
PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:/usr/bin/:/bin PATH = {{ buildout['bin-directory'] }}:{{ curl_location }}/bin/:{{ faketime_location }}/bin/:/usr/bin/:/bin
SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random} SLAPOS_TEST_IPV4 = ${slap-configuration:ipv4-random}
SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random} SLAPOS_TEST_IPV6 = ${slap-configuration:ipv6-random}
SLAPOS_TEST_WORKING_DIR = ${directory:working-dir} SLAPOS_TEST_WORKING_DIR = ${directory:working-dir}
......
...@@ -4,6 +4,7 @@ extends = ...@@ -4,6 +4,7 @@ extends =
../../component/bcrypt/buildout.cfg ../../component/bcrypt/buildout.cfg
../../component/curl/buildout.cfg ../../component/curl/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/faketime/buildout.cfg
../../component/pillow/buildout.cfg ../../component/pillow/buildout.cfg
../../component/python-cryptography/buildout.cfg ../../component/python-cryptography/buildout.cfg
../../component/python-mysqlclient/buildout.cfg ../../component/python-mysqlclient/buildout.cfg
...@@ -174,6 +175,7 @@ eggs = ...@@ -174,6 +175,7 @@ eggs =
${python-pynacl:egg} ${python-pynacl:egg}
${python-cryptography:egg} ${python-cryptography:egg}
${python-mysqlclient:egg} ${python-mysqlclient:egg}
${backports.lzma:egg}
${bcrypt:egg} ${bcrypt:egg}
slapos.libnetworkcache slapos.libnetworkcache
supervisor supervisor
...@@ -200,7 +202,6 @@ eggs = ...@@ -200,7 +202,6 @@ eggs =
${slapos.test.dream-setup:egg} ${slapos.test.dream-setup:egg}
${slapos.test.metabase-setup:egg} ${slapos.test.metabase-setup:egg}
${slapos.test.repman-setup:egg} ${slapos.test.repman-setup:egg}
${backports.lzma:egg}
entry-points = entry-points =
runTestSuite=erp5.util.testsuite:runTestSuite runTestSuite=erp5.util.testsuite:runTestSuite
scripts = scripts =
...@@ -239,6 +240,7 @@ context = ...@@ -239,6 +240,7 @@ context =
key slapos_location slapos-repository:location key slapos_location slapos-repository:location
key interpreter eggs:interpreter key interpreter eggs:interpreter
key curl_location curl:location key curl_location curl:location
key faketime_location faketime:location
key tests :tests key tests :tests
tests = tests =
${slapos.test.kvm-setup:setup} ${slapos.test.kvm-setup:setup}
...@@ -309,8 +311,6 @@ Django = 1.11 ...@@ -309,8 +311,6 @@ Django = 1.11
# selenium==3.141.0 # selenium==3.141.0
urllib3 = 1.24.1 urllib3 = 1.24.1
backports.lzma = 0.0.13
mock = 2.0.0 mock = 2.0.0
testfixtures = 6.11 testfixtures = 6.11
funcsigs = 1.0.2 funcsigs = 1.0.2
......
...@@ -57,7 +57,9 @@ depends = ${slapos.core-setup:egg} ...@@ -57,7 +57,9 @@ depends = ${slapos.core-setup:egg}
<= setup-develop-egg <= setup-develop-egg
egg = slapos.cookbook[test] egg = slapos.cookbook[test]
setup = ${slapos.cookbook-repository:location} setup = ${slapos.cookbook-repository:location}
depends = ${slapos.core-setup:egg} depends =
${slapos.core-setup:egg}
${slapos-cookbook-dependencies:eggs}
[slapos.core-setup] [slapos.core-setup]
<= setup-develop-egg <= setup-develop-egg
...@@ -83,10 +85,11 @@ setup = ${slapos.recipe.template-repository:location} ...@@ -83,10 +85,11 @@ setup = ${slapos.recipe.template-repository:location}
[slapos.toolbox-setup] [slapos.toolbox-setup]
<= setup-develop-egg <= setup-develop-egg
# XXX slapos.toolbox does not have `test` extra require, `mock` and `pycurl` are only listed in `tests_require` and are listed explicitly egg = slapos.toolbox[test]
egg = slapos.toolbox
setup = ${slapos.toolbox-repository:location} setup = ${slapos.toolbox-repository:location}
depends = ${slapos.core-setup:egg} depends =
${slapos.core-setup:egg}
${slapos-toolbox-dependencies:eggs}
[slapos.rebootstrap-setup] [slapos.rebootstrap-setup]
<= setup-develop-egg <= setup-develop-egg
...@@ -98,6 +101,7 @@ recipe = zc.recipe.egg ...@@ -98,6 +101,7 @@ recipe = zc.recipe.egg
eggs = eggs =
${lxml-python:egg} ${lxml-python:egg}
${python-cryptography:egg} ${python-cryptography:egg}
${backports.lzma:egg}
${pycurl:egg} ${pycurl:egg}
${bcrypt:egg} ${bcrypt:egg}
dnspython dnspython
...@@ -113,7 +117,6 @@ eggs = ...@@ -113,7 +117,6 @@ eggs =
${slapos.toolbox-setup:egg} ${slapos.toolbox-setup:egg}
${slapos.libnetworkcache-setup:egg} ${slapos.libnetworkcache-setup:egg}
${slapos.rebootstrap-setup:egg} ${slapos.rebootstrap-setup:egg}
mock
zope.testing zope.testing
supervisor supervisor
entry-points = entry-points =
......
...@@ -9,6 +9,7 @@ extends = ...@@ -9,6 +9,7 @@ extends =
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/graphviz/buildout.cfg ../../component/graphviz/buildout.cfg
../../component/gzip/buildout.cfg ../../component/gzip/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/haproxy/buildout.cfg ../../component/haproxy/buildout.cfg
../../component/hookbox/buildout.cfg ../../component/hookbox/buildout.cfg
../../component/findutils/buildout.cfg ../../component/findutils/buildout.cfg
...@@ -178,6 +179,7 @@ context = ...@@ -178,6 +179,7 @@ context =
key erp5_location erp5:location key erp5_location erp5:location
key findutils_location findutils:location key findutils_location findutils:location
key gzip_location gzip:location key gzip_location gzip:location
key xz_utils_location xz-utils:location
key haproxy_location haproxy:location key haproxy_location haproxy:location
key instance_common_cfg instance-common:rendered key instance_common_cfg instance-common:rendered
key jsl_location jsl:location key jsl_location jsl:location
...@@ -560,14 +562,11 @@ setup = ${erp5:location} ...@@ -560,14 +562,11 @@ setup = ${erp5:location}
[zodbpack] [zodbpack]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
${lxml-python:egg}
${pycurl:egg}
${python-PyYAML:egg}
${python-cryptography:egg}
${python-cliff:egg}
slapos.toolbox[zodbpack] slapos.toolbox[zodbpack]
scripts = scripts =
zodbpack zodbpack
depends =
${slapos-toolbox-dependencies:eggs}
[versions] [versions]
# See ../../software/neoppod/software-common.cfg for versions common with NEO: # See ../../software/neoppod/software-common.cfg for versions common with NEO:
......
...@@ -18,7 +18,7 @@ md5sum = 85ce1e2f3d251aa435fef8118dca8a63 ...@@ -18,7 +18,7 @@ md5sum = 85ce1e2f3d251aa435fef8118dca8a63
[mariadb-slow-query-report-script] [mariadb-slow-query-report-script]
filename = mysql-querydigest.sh.in filename = mysql-querydigest.sh.in
md5sum = 0c0d98a68230cd0ad36046bb25b35f4a md5sum = 7b14d2b81f2c864e47682ddb03b1b663
[mariadb-start-clone-from-backup] [mariadb-start-clone-from-backup]
filename = instance-mariadb-start-clone-from-backup.sh.in filename = instance-mariadb-start-clone-from-backup.sh.in
...@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196 ...@@ -26,7 +26,7 @@ md5sum = d10b8e35b02b5391cf46bf0c7dbb1196
[template-mariadb] [template-mariadb]
filename = instance-mariadb.cfg.in filename = instance-mariadb.cfg.in
md5sum = bfed6ac56c3ba0e96be4c9474dac6f20 md5sum = 7d064777c1c4e7b275b255db4f4b1da9
[template-kumofs] [template-kumofs]
filename = instance-kumofs.cfg.in filename = instance-kumofs.cfg.in
...@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec ...@@ -70,7 +70,7 @@ md5sum = cc19560b9400cecbd23064d55c501eec
[template] [template]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 328ea2bb5f2bff18f8be8c541c01f260 md5sum = 5c5250112b87a3937f939028f9594b85
[monitor-template-dummy] [monitor-template-dummy]
filename = dummy.cfg filename = dummy.cfg
......
...@@ -326,17 +326,18 @@ context = ...@@ -326,17 +326,18 @@ context =
raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log raw slow_query_path ${directory:srv}/backup/logrotate/mariadb_slowquery.log
raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path} raw pt_query_exec ${binary-wrap-pt-digest:wrapper-path}
raw dash {{ parameter_dict['dash-location'] }}/bin/dash raw dash {{ parameter_dict['dash-location'] }}/bin/dash
raw xz {{ parameter_dict['xz-utils-location'] }}/bin/xz
key output_folder directory:slowquery key output_folder directory:slowquery
[slow-query-digest-parameters] {%if slapparameter_dict.get('max-slowqueries-threshold') and slapparameter_dict.get('slowest-query-threshold') %}
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
[{{ section('monitor-promise-slowquery-result') }}] [{{ section('monitor-promise-slowquery-result') }}]
<= monitor-promise-base <= monitor-promise-base
module = check_command_execute module = check_command_execute
name = check-slow-query-pt-digest-result.py name = check-slow-query-pt-digest-result.py
config-command = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_path "${directory:slowquery}" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${slow-query-digest-parameters:max_queries_threshold}" --slowest_query_threshold "${slow-query-digest-parameters:slowest_queries_threshold}" config-command = "{{ parameter_dict['promise-check-slow-queries-digest-result'] }}" --ptdigest_path "${directory:slowquery}" --status_file ${monitor-directory:private}/mariadb_slow_query.report.json --max_queries_threshold "${:max_queries_threshold}" --slowest_query_threshold "${:slowest_queries_threshold}"
max_queries_threshold = {{ slapparameter_dict['max-slowqueries-threshold'] }}
slowest_queries_threshold = {{ slapparameter_dict['slowest-query-threshold'] }}
{%-endif%}
[{{ section('promise-check-computer-memory') }}] [{{ section('promise-check-computer-memory') }}]
<= monitor-promise-base <= monitor-promise-base
......
...@@ -144,6 +144,7 @@ coreutils-location = {{ coreutils_location }} ...@@ -144,6 +144,7 @@ coreutils-location = {{ coreutils_location }}
dash-location = {{ dash_location }} dash-location = {{ dash_location }}
findutils-location = {{ findutils_location }} findutils-location = {{ findutils_location }}
gzip-location = {{ gzip_location }} gzip-location = {{ gzip_location }}
xz-utils-location = {{ xz_utils_location }}
mariadb-location = {{ mariadb_location }} mariadb-location = {{ mariadb_location }}
template-my-cnf = {{ template_my_cnf }} template-my-cnf = {{ template_my_cnf }}
template-mariadb-initial-setup = {{ template_mariadb_initial_setup }} template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
......
#!{{dash}} #!{{dash}}
# BEWARE: This file is operated by slapgrid # BEWARE: This file is operated by slapgrid
set -e
SLOW_QUERY_PATH='{{slow_query_path}}' SLOW_QUERY_PATH='{{slow_query_path}}'
OUTPUT_FOLDER='{{output_folder}}' OUTPUT_FOLDER='{{output_folder}}'
PT_QUERY_EXEC='{{pt_query_exec}}' PT_QUERY_EXEC='{{pt_query_exec}}'
...@@ -21,6 +23,6 @@ if [ ! -f "$SLOW_LOG" ]; then ...@@ -21,6 +23,6 @@ if [ ! -f "$SLOW_LOG" ]; then
exit 1 exit 1
fi fi
"$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE" && \ "$PT_QUERY_EXEC" "$SLOW_LOG" > "$OUTPUT_FILE"
echo "Report generated successfully." || \
echo "Report failed with code $?" {{ xz }} -9 "$OUTPUT_FILE"
...@@ -31,6 +31,7 @@ extends = ...@@ -31,6 +31,7 @@ extends =
../../component/perl/buildout.cfg ../../component/perl/buildout.cfg
../../component/sqlite3/buildout.cfg ../../component/sqlite3/buildout.cfg
../../component/stunnel/buildout.cfg ../../component/stunnel/buildout.cfg
../../component/xz-utils/buildout.cfg
../../component/zlib/buildout.cfg ../../component/zlib/buildout.cfg
../erp5/buildout.cfg ../erp5/buildout.cfg
../logrotate/buildout.cfg ../logrotate/buildout.cfg
...@@ -88,6 +89,7 @@ context = ...@@ -88,6 +89,7 @@ context =
key logrotate_location logrotate:location key logrotate_location logrotate:location
key logrotate_cfg template-logrotate-base:rendered key logrotate_cfg template-logrotate-base:rendered
key gzip_location gzip:location key gzip_location gzip:location
key xz_utils_location xz-utils:location
key stunnel_location stunnel:location key stunnel_location stunnel:location
key template_monitor monitor2-template:rendered key template_monitor monitor2-template:rendered
key mariadb_link_binary template-mariadb:link-binary key mariadb_link_binary template-mariadb:link-binary
...@@ -137,6 +139,4 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_} ...@@ -137,6 +139,4 @@ location = ${buildout:parts-directory}/${:_buildout_section_name_}
[eggs] [eggs]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = eggs =
${lxml-python:egg} ${slapos-toolbox:eggs}
${pycurl:egg}
slapos.toolbox
...@@ -14,7 +14,7 @@ ...@@ -14,7 +14,7 @@
# not need these here). # not need these here).
[instance] [instance]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = be63b936ed521edaead8e0770ac64621 md5sum = 5c953c0f5d3376718eb9c4030288647a
[instance-apache-php] [instance-apache-php]
filename = instance-apache-php.cfg.in filename = instance-apache-php.cfg.in
......
...@@ -78,6 +78,7 @@ coreutils-location = {{ coreutils_location }} ...@@ -78,6 +78,7 @@ coreutils-location = {{ coreutils_location }}
dash-location = {{ dash_location }} dash-location = {{ dash_location }}
findutils-location = {{ findutils_location }} findutils-location = {{ findutils_location }}
gzip-location = {{ gzip_location }} gzip-location = {{ gzip_location }}
xz-utils-location = {{ xz_utils_location }}
mariadb-location = {{ mariadb_location }} mariadb-location = {{ mariadb_location }}
template-my-cnf = {{ template_my_cnf }} template-my-cnf = {{ template_my_cnf }}
template-mariadb-initial-setup = {{ template_mariadb_initial_setup }} template-mariadb-initial-setup = {{ template_mariadb_initial_setup }}
......
...@@ -21,6 +21,7 @@ develop = true ...@@ -21,6 +21,7 @@ develop = true
recipe = zc.recipe.egg:develop recipe = zc.recipe.egg:develop
egg = slapos.toolbox egg = slapos.toolbox
setup = ${slapos.toolbox-repository:location} setup = ${slapos.toolbox-repository:location}
depends = ${slapos-toolbox-dependencies:eggs}
[slapos.cookbook-repository] [slapos.cookbook-repository]
develop = true develop = true
......
...@@ -23,6 +23,7 @@ extends = ...@@ -23,6 +23,7 @@ extends =
../component/defaults.cfg ../component/defaults.cfg
../component/git/buildout.cfg ../component/git/buildout.cfg
../component/lxml-python/buildout.cfg ../component/lxml-python/buildout.cfg
../component/python-backports-lzma/buildout.cfg
../component/python-cffi/buildout.cfg ../component/python-cffi/buildout.cfg
../component/python-cliff/buildout.cfg ../component/python-cliff/buildout.cfg
../component/python-cachecontrol/buildout.cfg ../component/python-cachecontrol/buildout.cfg
...@@ -88,6 +89,11 @@ setup = ${slapos.cookbook-repository:location} ...@@ -88,6 +89,11 @@ setup = ${slapos.cookbook-repository:location}
# Install slapos.cookbook containing all officials recipes # Install slapos.cookbook containing all officials recipes
[slapos-cookbook] [slapos-cookbook]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = slapos.cookbook
depends = ${slapos-cookbook-dependencies:eggs}
[slapos-cookbook-dependencies]
recipe = zc.recipe.egg
eggs = eggs =
${lxml-python:egg} ${lxml-python:egg}
${python-cffi:egg} ${python-cffi:egg}
...@@ -96,17 +102,23 @@ eggs = ...@@ -96,17 +102,23 @@ eggs =
${python-cliff:egg} ${python-cliff:egg}
${python-cryptography:egg} ${python-cryptography:egg}
${jsonschema:egg} ${jsonschema:egg}
pyOpenSSL
slapos.cookbook
# slapos.toolbox containing utilities # slapos.toolbox containing utilities
[slapos-toolbox] [slapos-toolbox]
recipe = zc.recipe.egg recipe = zc.recipe.egg
eggs = slapos.toolbox
depends = ${slapos-toolbox-dependencies:eggs}
[slapos-toolbox-dependencies]
# slapos.toolbox depends on eggs with C-extensions that needs to be
# installed with slapos provided libraries. This section installs all
# slapos.toolbox dependencies.
recipe = zc.recipe.egg
eggs = eggs =
${lxml-python:egg} ${lxml-python:egg}
${pycurl:egg} ${pycurl:egg}
${python-cryptography:egg} ${python-cryptography:egg}
slapos.toolbox ${backports.lzma:egg}
[jsonschema] [jsonschema]
recipe = zc.recipe.egg:custom recipe = zc.recipe.egg:custom
...@@ -135,6 +147,7 @@ MarkupSafe = 1.0 ...@@ -135,6 +147,7 @@ MarkupSafe = 1.0
PyYAML = 3.13 PyYAML = 3.13
Werkzeug = 0.12 Werkzeug = 0.12
asn1crypto = 1.3.0 asn1crypto = 1.3.0
backports.lzma = 0.0.14
cffi = 1.14.0 cffi = 1.14.0
click = 6.7 click = 6.7
cliff = 2.4.0 cliff = 2.4.0
...@@ -160,14 +173,14 @@ pytz = 2016.10 ...@@ -160,14 +173,14 @@ pytz = 2016.10
requests = 2.24.0 requests = 2.24.0
six = 1.12.0 six = 1.12.0
slapos.cookbook = 1.0.152 slapos.cookbook = 1.0.152
slapos.core = 1.6.0 slapos.core = 1.6.1
slapos.extension.strip = 0.4 slapos.extension.strip = 0.4
slapos.extension.shared = 1.0 slapos.extension.shared = 1.0
slapos.libnetworkcache = 0.20 slapos.libnetworkcache = 0.20
slapos.rebootstrap = 4.5 slapos.rebootstrap = 4.5
slapos.recipe.build = 0.45 slapos.recipe.build = 0.45
slapos.recipe.cmmi = 0.16 slapos.recipe.cmmi = 0.16
slapos.toolbox = 0.109 slapos.toolbox = 0.110
stevedore = 1.21.0 stevedore = 1.21.0
subprocess32 = 3.5.3 subprocess32 = 3.5.3
unicodecsv = 0.14.1 unicodecsv = 0.14.1
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment