slapconfiguration: Add .jsonschema recipe

[slapconfiguration]
<= slap-connection
recipe = slapos.cookbook:slapconfiguration.jsonschema
jsonschema = ${buildout:directory}/software.cfg.json

The recipe validates instance parameters against the JSON schema, and
automatically fills in values of parameters left unset from defaults
specified in the schema, both for the main instance and for every
shared instance, if any.

Validation error or conflicting defaults - when the instance validates
multiple combined schemas that each set different defaults - can occur.

In case of error in the main instance parameters, the recipe raises an
exception. Otherwise, the completed instance parameters are available
in `configuration`.

For shared instances, valid and conflict-free instances are
collected in `valid-shared-instance-list` as dicts in the form of:
```
{'reference': "...", 'parameters': {...}}
```
with parameters completed by defaults as appropriate.

Invalid shared instances are collected in `invalid-slave-instance-list`
as dicts in the form of:
```
{'reference': "...", 'parameters': {...}, 'errors': [...]}
```
with parameters left as-is without added defaults.
Co-authored-by: Thomas Gambier <thomas.gambier@nexedi.com>

setup.py

@@ -150,6 +150,7 @@ setup(name=name,
           'simplehttpserver = slapos.recipe.simplehttpserver:Recipe',
           'slapconfiguration = slapos.recipe.slapconfiguration:Recipe',
           'slapconfiguration.serialised = slapos.recipe.slapconfiguration:Serialised',
+          'slapconfiguration.jsonschema = slapos.recipe.slapconfiguration:JsonSchema',
           'slapconfiguration.jsondump = slapos.recipe.slapconfiguration:JsonDump',
           'squid = slapos.recipe.squid:Recipe',
           'sshkeys_authority = slapos.recipe.sshkeys_authority:Recipe',

slapos/recipe/slapconfiguration.py

@@ -29,13 +29,24 @@ import json
 import logging
 import os
 
+from collections import defaultdict
+from contextlib import contextmanager
+
+import jsonschema
 import slapos.slap
 from slapos.recipe.librecipe import unwrap
 import six
 from six.moves.configparser import RawConfigParser
 from netaddr import valid_ipv4, valid_ipv6
-from slapos.util import mkdir_p
+from slapos.util import (
+  mkdir_p,
+  SoftwareReleaseSchema,
+  SoftwareReleaseSerialisation,
+  SoftwareReleaseSchemaValidationError,
+  urljoin,
+)
 from slapos import format as slapformat
+from zc.buildout import UserError
 
 
 logger = logging.getLogger("slapos")
@@ -164,7 +175,6 @@ class Recipe(object):
       options['root-instance-title'] = parameter_dict.pop('root_instance_title',
                                             'UNKNOWN')
       options['instance-guid'] = computer_partition.getInstanceGuid()
-
       ipv4_set = set()
       v4_add = ipv4_set.add
       ipv6_set = set()
@@ -187,6 +197,10 @@ class Recipe(object):
               v6_add(ip)
           # XXX: emit warning on unknown address type ?
 
+      # XXX slapproxy is sending 'full_address_list' not 'full_ip_list' (like real slapos master)
+      # just pop this value for now. Remove this when slapproxy is fixed.
+      parameter_dict.pop('full_address_list', None)
+
       if 'full_ip_list' in parameter_dict:
         for item in parameter_dict.pop('full_ip_list'):
           if len(item) == 5:
@@ -201,6 +215,11 @@ class Recipe(object):
               if valid_ipv4(network):
                 route_net_add(network)
 
+      # validate the parameters (only when using JsonSchema recipe)
+      # after popping the custom values sent by slapos master
+      # but before adding the value from .slapos-resources file
+      parameter_dict = self._validateParameterDict(options,parameter_dict)
+
       options['ipv4'] = ipv4_set
       options['ipv6'] = ipv6_set
 
@@ -215,7 +234,7 @@ class Recipe(object):
       if storage_home and os.path.exists(storage_home) and \
                                   os.path.isdir(storage_home):
         for filename in os.listdir(storage_home):
-          storage_path = os.path.join(storage_home, filename, 
+          storage_path = os.path.join(storage_home, filename,
                                     options['slap-computer-partition-id'])
           if os.path.exists(storage_path) and os.path.isdir(storage_path):
             storage_link = os.path.join(instance_root, 'DATA', filename)
@@ -249,6 +268,9 @@ class Recipe(object):
       logger.debug(str(options))
       return self._expandParameterDict(options, parameter_dict)
 
+  def _validateParameterDict(self, options, parameter_dict):
+      return parameter_dict
+
   def _expandParameterDict(self, options, parameter_dict):
       options['configuration'] = parameter_dict
       return parameter_dict
@@ -263,6 +285,139 @@ class Serialised(Recipe):
       else:
           return {}
 
+
+class DefaultValidator(object):
+  def __init__(self, schema):
+    self.schema = schema
+    self.validatorfor = v = jsonschema.validators.validator_for(schema)
+    # Retain original properties validator
+    validate_properties = v.VALIDATORS["properties"]
+    # Define new properties validator
+    def collect_defaults(validator, properties, instance, schema):
+      # Call original properties validator
+      error = False
+      for e in validate_properties(validator, properties, instance, schema):
+        error = True
+        yield e
+      # Collect defaults if the instance validates this schema
+      if not error:
+        for key, subschema in properties.items():
+          if "default" in subschema:
+            try:
+              _, defaults = self.defaults[id(instance)]
+            except KeyError:
+              defaults = defaultdict(dict)
+              self.defaults[id(instance)] = instance, defaults
+            defaults[key][id(subschema)] = subschema["default"]
+    # Extend validator class with extended properties validator
+    kls = jsonschema.validators.extend(v, {"properties" : collect_defaults})
+    self.validator = kls(schema)
+
+  @contextmanager
+  def propagate(self):
+    # Workaround https://github.com/python-jsonschema/jsonschema/issues/994
+    # This only works if all $ref schemas have the same $schema as the first.
+    version = self.schema.get("$schema")
+    try:
+      if version is not None:
+        jsonschema.validators.validates(version)(type(self.validator))
+      yield
+    finally:
+      if version is not None:
+        jsonschema.validators.validates(version)(self.validatorfor)
+
+  def validate(self, instance):
+    # Initialise default collection
+    self.defaults = {}
+    # Validate instance
+    invalid = False
+    with self.propagate():
+      for error in self.validator.iter_errors(instance):
+        invalid = True
+        yield error
+    # Apply collected defaults - only to valid instances
+    if invalid:
+      return
+    for instance, defaults in self.defaults.values():
+      for key, defaultdict in defaults.items():
+        if key not in instance:
+          it = iter(defaultdict.values())
+          default = next(it)
+          if any(d != default for d in it):
+            raise UserError(
+              "Conflicting defaults for key %s: %r" % (key, defaultlist))
+          instance[key] = default
+
+
+class JsonSchema(Recipe):
+  """
+  Input:
+    jsonschema
+      JSON Schema for the SR.
+      All instance schemas must be available at the advertised relative paths.
+      Example:
+        ${buildout:directory}/software.cfg.json
+  """
+  def _schema(self, options):
+    path = options['jsonschema']
+    # because SoftwareReleaseSchema accepts only file:// paths
+    path = path if path.startswith('file://') else 'file://' + path
+    # because SoftwareReleaseSchema expects the SR url and adds .json
+    path = path[:-5] if path.endswith('.json') else path
+    return SoftwareReleaseSchema(path, options['slap-software-type'])
+
+  def _getSharedSchema(self, software_schema):
+    t = software_schema.software_type
+    software_json_dict = software_schema.getSoftwareSchema()
+    for type_dict in software_json_dict['software-type'].values():
+      if type_dict['software-type'] == t and type_dict.get('shared') == True:
+        url = urljoin(software_schema.software_url, type_dict['request'])
+        return software_schema._readAsJson(url, True)
+
+  def _parseParameterDict(self, software_schema, parameter_dict):
+    instance_schema = software_schema.getInstanceRequestParameterSchema()
+    instance = parameter_dict if isinstance(parameter_dict, dict) else {}
+    validator = DefaultValidator(instance_schema)
+    errors = list(validator.validate(instance))
+    if errors:
+      err = SoftwareReleaseSchemaValidationError(errors).format_error(indent=2)
+      msg = "Invalid parameters:\n" + err
+      raise UserError(msg)
+    return instance
+
+  def _parseSharedParameterDict(self, software_schema, options):
+    shared_list = options.pop('slave-instance-list')
+    if not shared_list:
+      return
+    shared_schema = self._getSharedSchema(software_schema)
+    validator = DefaultValidator(shared_schema)
+    valid, invalid = [], []
+    for instance in shared_list:
+      reference = instance.pop('slave_reference')
+      try:
+        errors = list(validator.validate(instance))
+      except UserError as e:
+        errors = list(e.args)
+      shared_item = {'reference': reference, 'parameters': instance}
+      if errors:
+        shared_item['errors'] = errors
+        invalid.append(shared_item)
+      else:
+        valid.append(shared_item)
+    options['valid-shared-instance-list'] = valid
+    options['invalid-shared-instance-list'] = invalid
+
+  def _validateParameterDict(self, options, parameter_dict):
+    software_schema = self._schema(options)
+    serialisation = software_schema.getSerialisation(strict=True)
+    if serialisation == SoftwareReleaseSerialisation.JsonInXml:
+      parameter_dict = unwrap(parameter_dict)
+    self._parseSharedParameterDict(software_schema, options)
+    parameter_dict = self._parseParameterDict(software_schema, parameter_dict)
+    options['configuration'] = parameter_dict
+    return parameter_dict
+
+
 class JsonDump(Recipe):
   def __init__(self, buildout, name, options):
     parameter_dict = self.fetch_parameter_dict(options)