NEO: new private-tmpfs option to run DB & tests in RAM

slapos/recipe/neoppod.py

@@ -88,7 +88,9 @@ class NeoBaseRecipe(GenericBaseRecipe):
         )
     args += self._getOptionList()
     args += shlex.split(options.get('extra-options', ''))
-    return self.createWrapper(options['wrapper'], args)
+    private_tmpfs = self.parsePrivateTmpfs()
+    kw = {'private_tmpfs': private_tmpfs} if private_tmpfs else {}
+    return self.createWrapper(options['wrapper'], args, **kw)
 
   def _getBindingAddress(self):
     options = self.options

software/neoppod/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = 574acb0cae9af8ec2af52825fb2436d8
+md5sum = 512383220488335ac186013c2ffdc7c1
 
 [template-neo-my-cnf]
 filename = my.cnf.in
@@ -46,4 +46,4 @@ md5sum = 5afd326de385563b5aeac81039f23341
 
 [runTestSuite.in]
 _update_hash_filename_ = runTestSuite.in
-md5sum = d724ca0d54dda15bba68b8a87f0e4544
+md5sum = 7a0d5d259eb7f90fc0421d1264fbe7b5

software/neoppod/instance-neo-input-schema.json

@@ -114,6 +114,10 @@
               ],
               "type": "string"
             },
+            "private-tmpfs": {
+              "description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.",
+              "type": "string"
+            },
             "mysql": {
               "description": "Dictionary containing parameters for MySQL.",
               "default": {},

software/neoppod/instance-neo.cfg.in

@@ -2,22 +2,36 @@
 {% set part_list = [] -%}
 {% set init_list = [] -%}
 
+{% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%}
 {% set storage_type = slapparameter_dict.get('storage-type') or (
      'MySQL' if mariadb_location is defined else 'SQLite') -%}
 {% set mysql = storage_type == 'MySQL' -%}
 {% if mysql -%}
 
 [{{ section('mysqld') }}]
+{% if private_tmpfs -%}
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:etc_run}/mariadb
+private-tmpfs = {{ private_tmpfs }} ${my-cnf-parameters:tmp-directory}
+command-line = ${mariadb-ns:rendered}
+[mariadb-ns]
+rendered = ${directory:bin}/mariadb-ns
+{% else -%}
+rendered = ${directory:etc_run}/mariadb
+{% endif -%}
 recipe = slapos.recipe.template:jinja2
 template = {{ template_mysqld_wrapper }}
-rendered = ${directory:etc_run}/mariadb
 context =
   key defaults_file my-cnf:rendered
   key datadir my-cnf-parameters:data-directory
 
 [my-cnf-parameters]
 socket = ${directory:var_run}/mariadb.sock
+{% if private_tmpfs -%}
+data-directory = ${:tmp-directory}/mariadb
+{% else -%}
 data-directory = ${directory:srv}/mariadb
+{% endif -%}
 tmp-directory = ${directory:tmp}
 pid-file = ${directory:var_run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
@@ -105,6 +119,9 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
 {%- if mysql %}
 {%-   do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~  i ~ ';') %}
 database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
+{%- elif private_tmpfs %}
+private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
+database-parameters = ${directory:tmp}/db.sqlite
 {%- else %}
 database-parameters = ${directory:db-{{i}}}/db.sqlite
 
@@ -148,6 +165,14 @@ post = ${binary-wrap-mysql:command-line} -e "FLUSH LOGS"
 {% if runTestSuite_in is defined -%}
 # bin/runTestSuite to run NEO tests
 [{{ section('runTestSuite') }}]
+{%- if private_tmpfs %}
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:bin}/${:_buildout_section_name_}
+private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
+command-line = ${runTestSuite-ns:rendered}
+
+[runTestSuite-ns]
+{%- endif %}
 recipe = slapos.recipe.template:jinja2
 rendered = ${directory:bin}/${:_buildout_section_name_}
 template = {{ runTestSuite_in }}
@@ -157,6 +182,13 @@ context =
     section my_cnf_parameters my-cnf-parameters
     raw     bin_directory     {{ bin_directory }}
     raw     prepend_path      {{ mariadb_location }}/bin
+{%- if private_tmpfs %}
+    key     datadir           my-cnf-parameters:data-directory
+    key     results_directory directory:results
+
+[directory]
+results = ${directory:srv}/tests
+{%- endif %}
 {%- endif %}
 
 {%- endif %}

software/neoppod/runTestSuite.in

@@ -2,7 +2,7 @@
 """
   Script to run NEO test suite using Nexedi's test node framework.
 """
-import argparse, json, os, re, shutil, subprocess, sys, traceback
+import argparse, errno, json, os, re, shutil, subprocess, sys, traceback
 from erp5.util import taskdistribution
 from time import gmtime, sleep, strftime, time
 
@@ -130,6 +130,14 @@ def main():
           if timeout < time():
             raise RuntimeError("MySQL server not started")
           sleep(1)
+{%- if datadir is defined %}
+        # fake path for neostorage (getTopologyPath)
+        try:
+          os.mkdir({{ repr(datadir) }})
+        except OSError as e:
+          if e.errno != errno.EEXIST:
+            raise
+{%- endif %}
       with open(os.devnull) as stdin:
         p = subprocess.Popen(args, stdin=stdin, stdout=subprocess.PIPE,
                              stderr=subprocess.PIPE, env=env)
@@ -161,6 +169,13 @@ def main():
       date = strftime("%Y/%m/%d %H:%M:%S", gmtime(end)),
       stderr=stderr,
       **status_dict)
+{%- if results_directory is defined %}
+
+    results = {{ repr(results_directory + '/') }} + adapter
+    if os.path.exists(results):
+      shutil.rmtree(results)
+    shutil.move(temp, results)
+{%- endif %}
 
 if __name__ == "__main__":
     main()