WIP request parameter file + check schema

setup.py

@@ -74,6 +74,8 @@ setup(name=name,
           'six',
           'cachecontrol',
           'lockfile',
+          'jsonschema',
+          'pyaml',
           'uritemplate', # used by hateoas navigator
           'subprocess32; python_version<"3"'
         ] + additional_install_requires,

slapos/cli/request.py

@@ -29,13 +29,42 @@
 
 import pprint
 import os.path
+import argparse
+import json
 
 from slapos.cli.config import ClientConfigCommand
 from slapos.client import init, ClientConfig, _getSoftwareReleaseFromSoftwareString
 from slapos.slap import ResourceNotReady
+from slapos.util import SoftwareReleaseSchema
+from slapos.util import SoftwareReleaseSerialisation
+
+from typing import IO, Dict
+
+
+def getParametersFromFile(file, serialisation):
+  # type: (IO[str], str) -> Dict
+  if os.path.splitext(file.name)[1] == '.yaml':
+    import yaml
+    params = yaml.full_load(file)
+    if serialisation == 'json-in-xml':
+      params = {'_': json.dumps(params)}
+  elif os.path.splitext(file.name)[1] == '.xml':
+    import lxml.etree
+    tree = lxml.etree.parse(file)
+    params = {e.attrib['id']: e.text for e in tree.findall('/parameter')}
+    # because the use case of xml files is to copy paste existing XML parameters
+    # as found on slapos interface, we don't be clever regarding the serialisation
+    # and assume they are already correct.
+  else:
+    params = json.load(file)
+    if serialisation == 'json-in-xml':
+      params = {'_': json.dumps(params)}
+
+  return params
 
 
 def parse_option_dict(options):
+    # type: (str) -> Dict
     """
     Parse a list of option strings like foo=bar baz=qux and return a dictionary.
     Will raise if keys are repeated.
@@ -85,9 +114,11 @@ class RequestCommand(ClientConfigCommand):
 
         ap.add_argument('--parameters',
                         nargs='+',
-                        help="Instance parameters, in the form 'option1=value1 option2=value2'.\n"
-                        "The content of a file can also be passed as option=@filename")
+                        help="Instance parameters, in the form 'option1=value1 option2=value2'.")
 
+        ap.add_argument('--parameters-file',
+                        type=argparse.FileType('r'),
+                        help="Instance parameters, in a file.")
         return ap
 
     def take_action(self, args):
@@ -110,11 +141,17 @@ def do_request(logger, conf, local):
 
     if conf.software_url in local:
         conf.software_url = local[conf.software_url]
+
+    software_schema = SoftwareReleaseSchema(conf.software_url, conf.type)
+    software_schema_serialisation = software_schema.getSerialisation()
+    parameters = conf.parameters
+    if conf.parameters_file:
+      parameters = getParametersFromFile(conf.parameters_file, software_schema_serialisation)
     try:
         partition = local['slap'].registerOpenOrder().request(
             software_release=conf.software_url,
             partition_reference=conf.reference,
-            partition_parameter_kw=conf.parameters,
+            partition_parameter_kw=parameters,
             software_type=conf.type,
             filter_kw=conf.node,
             state=conf.state,
@@ -122,7 +159,11 @@ def do_request(logger, conf, local):
         )
         logger.info('Instance requested.\nState is : %s.', partition.getState())
         logger.info('Connection parameters of instance are:')
-        logger.info(pprint.pformat(partition.getConnectionParameterDict()))
+        connection_parameter_dict = partition.getConnectionParameterDict()
+        if software_schema_serialisation == SoftwareReleaseSerialisation.JsonInXml:
+          if '_' in connection_parameter_dict:
+            connection_parameter_dict = json.loads(connection_parameter_dict['_'])
+        logger.info(pprint.pformat(connection_parameter_dict))
         logger.info('You can rerun the command to get up-to-date information.')
     except ResourceNotReady:
         logger.warning('Instance requested. Master is provisioning it. Please rerun in a '

slapos/slap/slap.py

@@ -266,6 +266,20 @@ class OpenOrder(SlapRequester):
         'state': dumps(state),
         'shared_xml': dumps(shared),
     }
+
+    import warnings
+    import jsonschema
+    from ..util import SoftwareReleaseSchema
+    try:
+      SoftwareReleaseSchema(
+          request_dict['software_release'],
+          request_dict['software_type']
+      ).validateInstanceParameterDict(partition_parameter_kw)
+    except jsonschema.ValidationError as e:
+      warnings.warn(
+        "Request parameters do not validate against schema definition:\n{e}".format(e=e),
+        UserWarning,
+      )
     return self._requestComputerPartition(request_dict)
 
   def getInformation(self, partition_reference):

slapos/util.py

@@ -295,3 +295,152 @@ def rmtree(path):
     raise e  # XXX make pylint happy
 
   shutil.rmtree(path, onerror=chmod_retry)
+
+
+from six.moves.urllib_parse import urljoin
+import json
+import requests
+from typing import Dict, Optional, IO
+import warnings
+import jsonschema
+
+
+def _readAsJson(url):
+  # type: (str) -> Optional[Dict]
+  """Reads and parse the json file located at `url`.
+
+  `url` can also be the path of a local file.
+  """
+  if url.startswith('file://'):
+    url = url[len('file://'):]
+  path = url if os.path.exists(url) else None
+
+  if path:
+    with open(path) as f:
+      try:
+        return json.load(f)
+      except ValueError:
+        return None
+  if url.startswith('http://') or url.startswith('https://'):
+    try:
+      r = requests.get(url)
+      r.raise_for_status()
+      return r.json()
+    except (requests.exceptions.HTTPError, ValueError):
+      return None
+  return None
+
+
+import enum
+class SoftwareReleaseSerialisation(str, enum.Enum):
+  Xml = 'xml'
+  JsonInXml = 'json-in-xml'
+
+
+class SoftwareReleaseSchema(object):
+  def __init__(self, software_url, software_type):
+    # type: (str, Optional[str]) ->  None
+    self.software_url = software_url
+    self.software_type = software_type
+
+  def getSoftwareSchema(self):
+    # type: () -> Optional[Dict]
+    """Returns the schema for this software.
+    """
+    return _readAsJson(self.software_url + '.json')
+
+  def getSoftwareTypeSchema(self):
+    # type: () -> Optional[Dict]
+    """Returns schema for this software type.
+    """
+    software_schema = self.getSoftwareSchema()
+    if software_schema is None:
+      return None
+
+    software_type = self.software_type
+    from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE # TODO
+    if software_type is None:
+      software_type = DEFAULT_SOFTWARE_TYPE
+    # XXX some software are using "default" for default software type
+    if software_type == DEFAULT_SOFTWARE_TYPE \
+        and software_type not in software_schema['software-type'] \
+        and 'default' in software_schema['software-type']:
+      warnings.warn(
+          "Software release {} does not have schema for DEFAULT_SOFTWARE_TYPE but has one for 'default'."
+          " Using 'default' instead.".format(self.software_url),
+          UserWarning,
+      )
+      software_type = 'default'
+    return software_schema['software-type'].get(software_type) 
+
+  def getSerialisation(self):
+    # type: () -> Optional[SoftwareReleaseSerialisation]
+    """Returns the serialisation method used for parameters.
+    """
+    software_schema = self.getSoftwareSchema()
+    if software_schema is None:
+      return None
+    # TODO: return an instance with methods to actually serialize / deserialize ?
+    return SoftwareReleaseSerialisation(software_schema['serialisation'])
+
+  def getInstanceRequestParameterSchemaURL(self):
+    # type: () -> Optional[str]
+    """Returns the URL of the schema defining instance parameters.
+    """
+    software_type_schema = self.getSoftwareTypeSchema()
+    if software_type_schema is None:
+      return None
+    software_url = self.software_url
+    if os.path.exists(software_url):
+      software_url = 'file://' + software_url
+    return urljoin(software_url, software_type_schema['request'])
+ 
+  def getInstanceRequestParameterSchema(self):
+    # type: () -> Optional[Dict]
+    """Returns the schema defining instance parameters.
+    """
+    instance_parameter_schema_url = self.getInstanceRequestParameterSchemaURL()
+    if instance_parameter_schema_url is None:
+      return None
+    schema = _readAsJson(instance_parameter_schema_url)
+    if schema:
+      # so that jsonschema knows how to resolve references
+      schema.setdefault('$id', instance_parameter_schema_url)
+    return schema
+
+  def getInstanceConnectionParameterSchemaURL(self):
+    # type: () -> Optional[str]
+    """Returns the URL of the schema defining connection parameters published by the instance.
+    """
+    software_type_schema = self.getSoftwareTypeSchema()
+    if software_type_schema is None:
+      return None
+    return urljoin(self.software_url, software_type_schema['response'])
+
+  def getInstanceConnectionParameterSchema(self):
+    # type: () -> Optional[Dict]
+    """Returns the schema defining connection parameters published by the instance.
+    """
+    instance_parameter_schema_url = self.getInstanceConnectionParameterSchemaURL()
+    if instance_parameter_schema_url is None:
+      return None
+    schema = _readAsJson(instance_parameter_schema_url)
+    if schema:
+      # so that jsonschema knows how to resolve references
+      schema.setdefault('$id', instance_parameter_schema_url)
+    return schema
+
+  def validateInstanceParameterDict(self, parameter_dict):
+    # type: (Dict) -> None
+    """Validate instance parameters against the software schema.
+
+    Raise jsonschema.ValidationError if parameters does not validate.
+    """
+    schema_url = self.getInstanceRequestParameterSchemaURL()
+    if schema_url:
+      instance = parameter_dict if self.getSerialisation() == SoftwareReleaseSerialisation.Xml else json.loads(parameter_dict['_'])
+      instance.pop('$schema', None)
+      jsonschema.validate(
+          instance=instance,
+          schema=self.getInstanceRequestParameterSchema(),
+      )