NEO: review schema and add support for SQLite configuration

See neoppod@71564067

software/neoppod/buildout.hash.cfg

@@ -14,7 +14,7 @@
 # not need these here).
 [instance-common]
 filename = instance-common.cfg.in
-md5sum = e000e7134113b9d1c63d40861eaf0489
+md5sum = ecc98da90cd446ea224ddeece1374190
 
 [root-common]
 filename = root-common.cfg.in
@@ -30,7 +30,7 @@ md5sum = 9f27195d770b2f57461c60a82c851ab9
 
 [instance-neo]
 filename = instance-neo.cfg.in
-md5sum = fda911d5ef9efee365f1b0ff9843a50b
+md5sum = 200ae55715cb735b0f97f8c835a3071f
 
 [template-neo-my-cnf]
 filename = my.cnf.in

software/neoppod/instance-common.cfg.in

@@ -41,6 +41,7 @@ url = {{ neo_master }}
 <= jinja2-template-base
 url = {{ neo }}
 extra-context =
+    import urllib urllib
     key master_cfg neo-master:output
     key admin_cfg neo-admin:output
 {%- if mariadb_location is defined %}

software/neoppod/instance-neo-input-schema.json

@@ -110,14 +110,6 @@
               "default": false,
               "type": "boolean"
             },
-            "storage-type": {
-              "description": "Storage type. Defaults to MySQL if available, else SQLite.",
-              "enum": [
-                "MySQL",
-                "SQLite"
-              ],
-              "type": "string"
-            },
             "private-tmpfs": {
               "description": "Size of private tmpfs mount to store the database. See filesystems/tmpfs.txt in Linux documentation. Use only for testing.",
               "type": "string"
@@ -126,25 +118,49 @@
               "description": "List of bindings to test when running the test suite.",
               "type": "array"
             },
+            "storage-type": {
+              "description": "Storage type. Required when several types are configured and you select which one to use via 'node!' parameter. Defaults to whatever is configured ('sqlite' or 'mysql'), else MySQL if available, else SQLite.",
+              "enum": [
+                "MySQL",
+                "SQLite"
+              ]
+            },
+            "sqlite": {
+              "description": "Storage backend configuration.",
+              "properties": {
+                "relaxed-writes": {
+                  "description": "When enabled, sets synchronous = OFF and journal_mode = MEMORY - RTFM, those options are dangerous.",
+                  "default": false,
+                  "type": "boolean"
+                }
+              },
+              "additionalProperties": {
+                "description": "See NEO documentation for the list of supported settings.",
+                "type": ["number", "string"]
+              },
+              "type": "object"
+            },
             "mysql": {
-              "description": "Dictionary containing parameters for MySQL.",
-              "default": {},
+              "description": "MariaDB server configuration.",
               "properties": {
                 "relaxed-writes": {
-                  "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous",
+                  "description": "When enabled, sets innodb_flush_log_at_trx_commit = 0, innodb_flush_method = nosync, innodb_doublewrite = 0 and sync_frm = 0 - RTFM, those options are dangerous.",
                   "default": false,
                   "type": "boolean"
                 }
               },
               "additionalProperties": {
-                "description": "To configure important parameters like innodb_buffer_pool_size, rocksdb_block_cache_size, etc.",
-                "type": "string"
+                "description": "To configure parameters like innodb_buffer_pool_size, rocksdb_block_cache_size, etc.",
+                "type": ["number", "string"]
               },
               "type": "object"
             },
             "engine": {
-              "description": "Configures storage engine, currently only InnoDB and RocksDB are supported. Defaults to NEO's default.",
-              "type": "string"
+              "description": "[MySQL only] For NEO, this is a creation-time parameter and it defaults to NEO's default. For mysqld, this sets plugins to load and it defaults to load all.",
+              "enum": [
+                "InnoDB",
+                "RocksDB"
+              ]
             }
           },
           "type": "object"

software/neoppod/instance-neo.cfg.in

 {% macro section(name) %}{% do part_list.append(name) %}{{ name }}{% endmacro -%}
 {% set part_list = [] -%}
 {% set init_list = [] -%}
+{% set directory_dict = {} -%}
 
 {% set private_tmpfs = slapparameter_dict.get('private-tmpfs') -%}
 {% set storage_count = slapparameter_dict.get('storage-count', 1) -%}
-{% set storage_type = slapparameter_dict.get('storage-type') or (
-     'MySQL' if mariadb_location is defined else 'SQLite') -%}
-{% set mysql = storage_count and storage_type != 'SQLite' -%}
+{# When mixing different storage types via node specialisation
+   ('node!' parameter), it can be convenient to configure all types in the
+   common 'node' parameter and then switch between the 2 with 'storage-type'.
+   So we must be quite tolerant. -#}
+{% if storage_count -%}
+{%   if 'mysql' in slapparameter_dict -%}
+{%     if 'sqlite' in slapparameter_dict -%}
+{%       set storage_type = slapparameter_dict['storage-type'] -%}
+{%     else -%}
+{%       set storage_type = 'MySQL' -%}
+{%     endif -%}
+{%   elif 'sqlite' in slapparameter_dict -%}
+{%     set storage_type = 'SQLite' -%}
+{%   else -%}
+{%     set storage_type = slapparameter_dict.get('storage-type') or (
+         'MySQL' if mariadb_location is defined else 'SQLite') -%}
+{%   endif -%}
+{%   do assert(slapparameter_dict.get('storage-type', storage_type) == storage_type) -%}
+{% else -%}
+{%   set storage_type = '' -%}
+{% endif -%}
+
+{% set mysql = storage_type == 'MySQL' -%}
 {% if mysql -%}
+{%   set extra_dict = slapparameter_dict.get('mysql') or {} -%}
 
 [{{ section('mysqld') }}]
 {% if private_tmpfs -%}
@@ -37,7 +59,7 @@ tmp-directory = ${directory:tmp}
 pid-file = ${directory:var_run}/mariadb.pid
 error-log = ${directory:log}/mariadb_error.log
 slow-query-log = ${directory:log}/mariadb_slowquery.log
-extra-dict = {{ dumps(slapparameter_dict.get('mysql', {})) }}
+extra-dict = {{ dumps(extra_dict) }}
 init-file = ${init-script:output}
 engine = {{ slapparameter_dict.get('engine', '') }}
 
@@ -54,6 +76,18 @@ command-line = '{{ mariadb_location }}/bin/${:command}' --defaults-file="${my-cn
 wrapper-path = ${directory:bin}/${:command}
 command = mysql
 
+{% elif storage_type == 'SQLite' -%}
+{%   set extra_dict = slapparameter_dict.get('sqlite') or {} -%}
+
+{%   if extra_dict.pop('relaxed-writes', False) -%}
+{%     do extra_dict.setdefault('synchronous', 'OFF') -%}
+{%     do extra_dict.setdefault('journal_mode', 'MEMORY') -%}
+{%   endif -%}
+{%   set query_string = urllib.urlencode(extra_dict) -%}
+
+{% else -%}
+{% do assert(not storage_count) -%}
+
 {% endif -%}
 
 [{{ section('binary-neolog') }}]
@@ -123,14 +157,19 @@ logfile = ${directory:log}/{{ 'neostorage-' ~ i }}.log
 {%- if mysql %}
 {%-   do init_list.append('CREATE DATABASE IF NOT EXISTS neo' ~  i ~ ';') %}
 database-parameters = root@neo{{ i }}${my-cnf-parameters:socket}
-{%- elif private_tmpfs %}
-private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
-database-parameters = ${directory:tmp}/db.sqlite
 {%- else %}
-database-parameters = ${directory:db-{{i}}}/db.sqlite
-
-[directory]
-db-{{i}} = ${:srv}/{{ storage_id }}
+{%-   if private_tmpfs %}
+private-tmpfs = {{ private_tmpfs }} ${directory:tmp}
+{%-     set path = '${directory:tmp}/db.sqlite' %}
+{%-   else %}
+{%-     set path = '${directory:db-' ~ i ~ '}/db.sqlite' %}
+{%-     do directory_dict.__setitem__('db-' ~ i, '${:srv}/' + storage_id) %}
+{%-   endif %}
+{%-   if query_string %}
+database-parameters = file:{{ path }}?{{ query_string }}
+{%-   else %}
+database-parameters = {{ path }}
+{%-   endif -%}
 {%- endif %}
 
 [{{ section('logrotate-storage-' ~ i) }}]
@@ -141,17 +180,6 @@ post = {{ bin_directory }}/slapos-kill -s RTMIN+1 -- {{ bin_directory }}/neostor
 
 {% endfor -%}
 
-[directory]
-recipe = slapos.cookbook:mkdirectory
-bin = ${buildout:directory}/bin
-etc = ${buildout:directory}/etc
-var = ${buildout:directory}/var
-etc_run =  ${:etc}/run
-var_run =  ${:var}/run
-log = ${buildout:directory}/var/log
-tmp = ${buildout:directory}/tmp
-srv = ${buildout:directory}/srv
-
 {% if mysql -%}
 [init-script]
 recipe = slapos.recipe.template
@@ -196,13 +224,26 @@ context =
     key     datadir           my-cnf-parameters:data-directory
     key     results_directory directory:results
 
-[directory]
-results = ${directory:srv}/tests
+{%-    do directory_dict.__setitem__('results', '${directory:srv}/tests') %}
 {%-  endif %}
 {%- endif %}
 
 {%- endif %}
 
+[directory]
+recipe = slapos.cookbook:mkdirectory
+bin = ${buildout:directory}/bin
+etc = ${buildout:directory}/etc
+var = ${buildout:directory}/var
+etc_run =  ${:etc}/run
+var_run =  ${:var}/run
+log = ${buildout:directory}/var/log
+tmp = ${buildout:directory}/tmp
+srv = ${buildout:directory}/srv
+{%- for k, v in directory_dict.iteritems() %}
+{{ k }} = {{ v }}
+{%- endfor %}
+
 [buildout]
 extends =
   {{ logrotate_cfg }}