Commit e354e214 authored by Julien Muchembled's avatar Julien Muchembled

NEO: new private-tmpfs option to run DB & tests in RAM

parent 28ea60be
Pipeline #8202 failed with stage
...@@ -88,7 +88,9 @@ class NeoBaseRecipe(GenericBaseRecipe): ...@@ -88,7 +88,9 @@ class NeoBaseRecipe(GenericBaseRecipe):
) )
args += self._getOptionList() args += self._getOptionList()
args += shlex.split(options.get('extra-options', '')) args += shlex.split(options.get('extra-options', ''))
return self.createWrapper(options['wrapper'], args) private_tmpfs = self.parsePrivateTmpfs()
kw = {'private_tmpfs': private_tmpfs} if private_tmpfs else {}
return self.createWrapper(options['wrapper'], args, **kw)
def _getBindingAddress(self): def _getBindingAddress(self):
options = self.options options = self.options
......
...@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9 ...@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
[instance-neo] [instance-neo]
filename = instance-neo.cfg.in filename = instance-neo.cfg.in
md5sum = 574acb0cae9af8ec2af52825fb2436d8 md5sum = 512383220488335ac186013c2ffdc7c1
[template-neo-my-cnf] [template-neo-my-cnf]
filename = my.cnf.in filename = my.cnf.in
...@@ -46,4 +46,4 @@ md5sum = 5afd326de385563b5aeac81039f23341 ...@@ -46,4 +46,4 @@ md5sum = 5afd326de385563b5aeac81039f23341
[runTestSuite.in] [runTestSuite.in]
_update_hash_filename_ = runTestSuite.in _update_hash_filename_ = runTestSuite.in
md5sum = d724ca0d54dda15bba68b8a87f0e4544 md5sum = 7a0d5d259eb7f90fc0421d1264fbe7b5
...@@ -114,6 +114,10 @@ ...@@ -114,6 +114,10 @@
], ],
"type": "string" "type": "string"
}, },
"private-tmpfs": {
"description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.",
"type": "string"
},
"mysql": { "mysql": {
"description": "Dictionary containing parameters for MySQL.", "description": "Dictionary containing parameters for MySQL.",
"default": {}, "default": {},
......
...@@ -2,22 +2,36 @@ ...@@ -2,22 +2,36 @@
{% set part_list = [] -%} {% set part_list = [] -%}
{% set init_list = [] -%} {% set init_list = [] -%}
{% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%}
{% set storage_type = slapparameter_dict.get('storage-type') or ( {% set storage_type = slapparameter_dict.get('storage-type') or (
'MySQL' if mariadb_location is defined else 'SQLite') -%} 'MySQL' if mariadb_location is defined else 'SQLite') -%}
{% set mysql = storage_type == 'MySQL' -%} {% set mysql = storage_type == 'MySQL' -%}
{% if mysql -%} {% if mysql -%}
[{{ section('mysqld') }}] [{{ section('mysqld') }}]
{% if private_tmpfs -%}
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:etc_run}/mariadb
private-tmpfs = {{ private_tmpfs }} ${my-cnf-parameters:tmp-directory}
command-line = ${mariadb-ns:rendered}
[mariadb-ns]
rendered = ${directory:bin}/mariadb-ns
{% else -%}
rendered = ${directory:etc_run}/mariadb
{% endif -%}
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
template = {{ template_mysqld_wrapper }} template = {{ template_mysqld_wrapper }}
rendered = ${directory:etc_run}/mariadb
context = context =
key defaults_file my-cnf:rendered key defaults_file my-cnf:rendered
key datadir my-cnf-parameters:data-directory key datadir my-cnf-parameters:data-directory
[my-cnf-parameters] [my-cnf-parameters]
socket = ${directory:var_run}/mariadb.sock socket = ${directory:var_run}/mariadb.sock
{% if private_tmpfs -%}
data-directory = ${:tmp-directory}/mariadb
{% else -%}
data-directory = ${directory:srv}/mariadb data-directory = ${directory:srv}/mariadb
{% endif -%}
tmp-directory = ${directory:tmp} tmp-directory = ${directory:tmp}
pid-file = ${directory:var_run}/mariadb.pid pid-file = ${directory:var_run}/mariadb.pid
error-log = ${directory:log}/mariadb_error.log error-log = ${directory:log}/mariadb_error.log
...@@ -105,6 +119,9 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log ...@@ -105,6 +119,9 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
{%- if mysql %} {%- if mysql %}
{%- do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') %} {%- do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~ i ~ ';') %}
database-parameters = root@neo{{ i }}${my-cnf-parameters:socket} database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
{%- elif private_tmpfs %}
private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
database-parameters = ${directory:tmp}/db.sqlite
{%- else %} {%- else %}
database-parameters = ${directory:db-{{i}}}/db.sqlite database-parameters = ${directory:db-{{i}}}/db.sqlite
...@@ -148,6 +165,14 @@ post = ${binary-wrap-mysql:command-line} -e "FLUSH LOGS" ...@@ -148,6 +165,14 @@ post = ${binary-wrap-mysql:command-line} -e "FLUSH LOGS"
{% if runTestSuite_in is defined -%} {% if runTestSuite_in is defined -%}
# bin/runTestSuite to run NEO tests # bin/runTestSuite to run NEO tests
[{{ section('runTestSuite') }}] [{{ section('runTestSuite') }}]
{%- if private_tmpfs %}
recipe = slapos.cookbook:wrapper
wrapper-path = ${directory:bin}/${:_buildout_section_name_}
private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
command-line = ${runTestSuite-ns:rendered}
[runTestSuite-ns]
{%- endif %}
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
rendered = ${directory:bin}/${:_buildout_section_name_} rendered = ${directory:bin}/${:_buildout_section_name_}
template = {{ runTestSuite_in }} template = {{ runTestSuite_in }}
...@@ -157,6 +182,13 @@ context = ...@@ -157,6 +182,13 @@ context =
section my_cnf_parameters my-cnf-parameters section my_cnf_parameters my-cnf-parameters
raw bin_directory {{ bin_directory }} raw bin_directory {{ bin_directory }}
raw prepend_path {{ mariadb_location }}/bin raw prepend_path {{ mariadb_location }}/bin
{%- if private_tmpfs %}
key datadir my-cnf-parameters:data-directory
key results_directory directory:results
[directory]
results = ${directory:srv}/tests
{%- endif %}
{%- endif %} {%- endif %}
{%- endif %} {%- endif %}
......
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
""" """
Script to run NEO test suite using Nexedi's test node framework. Script to run NEO test suite using Nexedi's test node framework.
""" """
import argparse, json, os, re, shutil, subprocess, sys, traceback import argparse, errno, json, os, re, shutil, subprocess, sys, traceback
from erp5.util import taskdistribution from erp5.util import taskdistribution
from time import gmtime, sleep, strftime, time from time import gmtime, sleep, strftime, time
...@@ -130,6 +130,14 @@ def main(): ...@@ -130,6 +130,14 @@ def main():
if timeout < time(): if timeout < time():
raise RuntimeError("MySQL server not started") raise RuntimeError("MySQL server not started")
sleep(1) sleep(1)
{%- if datadir is defined %}
# fake path for neostorage (getTopologyPath)
try:
os.mkdir({{ repr(datadir) }})
except OSError as e:
if e.errno != errno.EEXIST:
raise
{%- endif %}
with open(os.devnull) as stdin: with open(os.devnull) as stdin:
p = subprocess.Popen(args, stdin=stdin, stdout=subprocess.PIPE, p = subprocess.Popen(args, stdin=stdin, stdout=subprocess.PIPE,
stderr=subprocess.PIPE, env=env) stderr=subprocess.PIPE, env=env)
...@@ -161,6 +169,13 @@ def main(): ...@@ -161,6 +169,13 @@ def main():
date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)), date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
stderr=stderr, stderr=stderr,
**status_dict) **status_dict)
{%- if results_directory is defined %}
results = {{ repr(results_directory + '/') }} + adapter
if os.path.exists(results):
shutil.rmtree(results)
shutil.move(temp, results)
{%- endif %}
if __name__ == "__main__": if __name__ == "__main__":
main() main()
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment