Commit 03f65757 authored by Jérome Perrin's avatar Jérome Perrin

WIP request parameter file + check schema

parent 34fcb64c
...@@ -74,6 +74,8 @@ setup(name=name, ...@@ -74,6 +74,8 @@ setup(name=name,
'six', 'six',
'cachecontrol', 'cachecontrol',
'lockfile', 'lockfile',
'jsonschema',
'pyaml',
'uritemplate', # used by hateoas navigator 'uritemplate', # used by hateoas navigator
'subprocess32; python_version<"3"' 'subprocess32; python_version<"3"'
] + additional_install_requires, ] + additional_install_requires,
......
...@@ -29,13 +29,42 @@ ...@@ -29,13 +29,42 @@
import pprint import pprint
import os.path import os.path
import argparse
import json
from slapos.cli.config import ClientConfigCommand from slapos.cli.config import ClientConfigCommand
from slapos.client import init, ClientConfig, _getSoftwareReleaseFromSoftwareString from slapos.client import init, ClientConfig, _getSoftwareReleaseFromSoftwareString
from slapos.slap import ResourceNotReady from slapos.slap import ResourceNotReady
from slapos.util import SoftwareReleaseSchema
from slapos.util import SoftwareReleaseSerialisation
from typing import IO, Dict
def getParametersFromFile(file, serialisation):
# type: (IO[str], str) -> Dict
if os.path.splitext(file.name)[1] == '.yaml':
import yaml
params = yaml.full_load(file)
if serialisation == 'json-in-xml':
params = {'_': json.dumps(params)}
elif os.path.splitext(file.name)[1] == '.xml':
import lxml.etree
tree = lxml.etree.parse(file)
params = {e.attrib['id']: e.text for e in tree.findall('/parameter')}
# because the use case of xml files is to copy paste existing XML parameters
# as found on slapos interface, we don't be clever regarding the serialisation
# and assume they are already correct.
else:
params = json.load(file)
if serialisation == 'json-in-xml':
params = {'_': json.dumps(params)}
return params
def parse_option_dict(options): def parse_option_dict(options):
# type: (str) -> Dict
""" """
Parse a list of option strings like foo=bar baz=qux and return a dictionary. Parse a list of option strings like foo=bar baz=qux and return a dictionary.
Will raise if keys are repeated. Will raise if keys are repeated.
...@@ -85,9 +114,11 @@ class RequestCommand(ClientConfigCommand): ...@@ -85,9 +114,11 @@ class RequestCommand(ClientConfigCommand):
ap.add_argument('--parameters', ap.add_argument('--parameters',
nargs='+', nargs='+',
help="Instance parameters, in the form 'option1=value1 option2=value2'.\n" help="Instance parameters, in the form 'option1=value1 option2=value2'.")
"The content of a file can also be passed as option=@filename")
ap.add_argument('--parameters-file',
type=argparse.FileType('r'),
help="Instance parameters, in a file.")
return ap return ap
def take_action(self, args): def take_action(self, args):
...@@ -110,11 +141,17 @@ def do_request(logger, conf, local): ...@@ -110,11 +141,17 @@ def do_request(logger, conf, local):
if conf.software_url in local: if conf.software_url in local:
conf.software_url = local[conf.software_url] conf.software_url = local[conf.software_url]
software_schema = SoftwareReleaseSchema(conf.software_url, conf.type)
software_schema_serialisation = software_schema.getSerialisation()
parameters = conf.parameters
if conf.parameters_file:
parameters = getParametersFromFile(conf.parameters_file, software_schema_serialisation)
try: try:
partition = local['slap'].registerOpenOrder().request( partition = local['slap'].registerOpenOrder().request(
software_release=conf.software_url, software_release=conf.software_url,
partition_reference=conf.reference, partition_reference=conf.reference,
partition_parameter_kw=conf.parameters, partition_parameter_kw=parameters,
software_type=conf.type, software_type=conf.type,
filter_kw=conf.node, filter_kw=conf.node,
state=conf.state, state=conf.state,
...@@ -122,7 +159,11 @@ def do_request(logger, conf, local): ...@@ -122,7 +159,11 @@ def do_request(logger, conf, local):
) )
logger.info('Instance requested.\nState is : %s.', partition.getState()) logger.info('Instance requested.\nState is : %s.', partition.getState())
logger.info('Connection parameters of instance are:') logger.info('Connection parameters of instance are:')
logger.info(pprint.pformat(partition.getConnectionParameterDict())) connection_parameter_dict = partition.getConnectionParameterDict()
if software_schema_serialisation == SoftwareReleaseSerialisation.JsonInXml:
if '_' in connection_parameter_dict:
connection_parameter_dict = json.loads(connection_parameter_dict['_'])
logger.info(pprint.pformat(connection_parameter_dict))
logger.info('You can rerun the command to get up-to-date information.') logger.info('You can rerun the command to get up-to-date information.')
except ResourceNotReady: except ResourceNotReady:
logger.warning('Instance requested. Master is provisioning it. Please rerun in a ' logger.warning('Instance requested. Master is provisioning it. Please rerun in a '
......
...@@ -266,6 +266,20 @@ class OpenOrder(SlapRequester): ...@@ -266,6 +266,20 @@ class OpenOrder(SlapRequester):
'state': dumps(state), 'state': dumps(state),
'shared_xml': dumps(shared), 'shared_xml': dumps(shared),
} }
import warnings
import jsonschema
from ..util import SoftwareReleaseSchema
try:
SoftwareReleaseSchema(
request_dict['software_release'],
request_dict['software_type']
).validateInstanceParameterDict(partition_parameter_kw)
except jsonschema.ValidationError as e:
warnings.warn(
"Request parameters do not validate against schema definition:\n{e}".format(e=e),
UserWarning,
)
return self._requestComputerPartition(request_dict) return self._requestComputerPartition(request_dict)
def getInformation(self, partition_reference): def getInformation(self, partition_reference):
......
...@@ -295,3 +295,152 @@ def rmtree(path): ...@@ -295,3 +295,152 @@ def rmtree(path):
raise e # XXX make pylint happy raise e # XXX make pylint happy
shutil.rmtree(path, onerror=chmod_retry) shutil.rmtree(path, onerror=chmod_retry)
from six.moves.urllib_parse import urljoin
import json
import requests
from typing import Dict, Optional, IO
import warnings
import jsonschema
def _readAsJson(url):
# type: (str) -> Optional[Dict]
"""Reads and parse the json file located at `url`.
`url` can also be the path of a local file.
"""
if url.startswith('file://'):
url = url[len('file://'):]
path = url if os.path.exists(url) else None
if path:
with open(path) as f:
try:
return json.load(f)
except ValueError:
return None
if url.startswith('http://') or url.startswith('https://'):
try:
r = requests.get(url)
r.raise_for_status()
return r.json()
except (requests.exceptions.HTTPError, ValueError):
return None
return None
import enum
class SoftwareReleaseSerialisation(str, enum.Enum):
Xml = 'xml'
JsonInXml = 'json-in-xml'
class SoftwareReleaseSchema(object):
def __init__(self, software_url, software_type):
# type: (str, Optional[str]) -> None
self.software_url = software_url
self.software_type = software_type
def getSoftwareSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema for this software.
"""
return _readAsJson(self.software_url + '.json')
def getSoftwareTypeSchema(self):
# type: () -> Optional[Dict]
"""Returns schema for this software type.
"""
software_schema = self.getSoftwareSchema()
if software_schema is None:
return None
software_type = self.software_type
from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE # TODO
if software_type is None:
software_type = DEFAULT_SOFTWARE_TYPE
# XXX some software are using "default" for default software type
if software_type == DEFAULT_SOFTWARE_TYPE \
and software_type not in software_schema['software-type'] \
and 'default' in software_schema['software-type']:
warnings.warn(
"Software release {} does not have schema for DEFAULT_SOFTWARE_TYPE but has one for 'default'."
" Using 'default' instead.".format(self.software_url),
UserWarning,
)
software_type = 'default'
return software_schema['software-type'].get(software_type)
def getSerialisation(self):
# type: () -> Optional[SoftwareReleaseSerialisation]
"""Returns the serialisation method used for parameters.
"""
software_schema = self.getSoftwareSchema()
if software_schema is None:
return None
# TODO: return an instance with methods to actually serialize / deserialize ?
return SoftwareReleaseSerialisation(software_schema['serialisation'])
def getInstanceRequestParameterSchemaURL(self):
# type: () -> Optional[str]
"""Returns the URL of the schema defining instance parameters.
"""
software_type_schema = self.getSoftwareTypeSchema()
if software_type_schema is None:
return None
software_url = self.software_url
if os.path.exists(software_url):
software_url = 'file://' + software_url
return urljoin(software_url, software_type_schema['request'])
def getInstanceRequestParameterSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema defining instance parameters.
"""
instance_parameter_schema_url = self.getInstanceRequestParameterSchemaURL()
if instance_parameter_schema_url is None:
return None
schema = _readAsJson(instance_parameter_schema_url)
if schema:
# so that jsonschema knows how to resolve references
schema.setdefault('$id', instance_parameter_schema_url)
return schema
def getInstanceConnectionParameterSchemaURL(self):
# type: () -> Optional[str]
"""Returns the URL of the schema defining connection parameters published by the instance.
"""
software_type_schema = self.getSoftwareTypeSchema()
if software_type_schema is None:
return None
return urljoin(self.software_url, software_type_schema['response'])
def getInstanceConnectionParameterSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema defining connection parameters published by the instance.
"""
instance_parameter_schema_url = self.getInstanceConnectionParameterSchemaURL()
if instance_parameter_schema_url is None:
return None
schema = _readAsJson(instance_parameter_schema_url)
if schema:
# so that jsonschema knows how to resolve references
schema.setdefault('$id', instance_parameter_schema_url)
return schema
def validateInstanceParameterDict(self, parameter_dict):
# type: (Dict) -> None
"""Validate instance parameters against the software schema.
Raise jsonschema.ValidationError if parameters does not validate.
"""
schema_url = self.getInstanceRequestParameterSchemaURL()
if schema_url:
instance = parameter_dict if self.getSerialisation() == SoftwareReleaseSerialisation.Xml else json.loads(parameter_dict['_'])
instance.pop('$schema', None)
jsonschema.validate(
instance=instance,
schema=self.getInstanceRequestParameterSchema(),
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment