Commit eb753808 authored by Jérome Perrin's avatar Jérome Perrin

cli/request: support passing instance parameters from a file

Support a syntax like:

      slapos request \
        --node=computer_guid=local \
        --parameters-file=~/request.json \
        ERP5 \
        https://lab.nexedi.com/nexedi/slapos/raw/1.0.145/software/erp5/software.cfg

to request an instance with parameters in ~/request.json file.

This also teach slapos.core about the instance schema, which makes it possible to
understand the serialisation, so that we no longer need to use `_ = json.dump(params)`
to encode parameters.

This also emits a warning when requesting with parameters which do not match the schema.


See merge request !215
parents a244d797 451c74d1
......@@ -74,6 +74,8 @@ setup(name=name,
'six',
'cachecontrol',
'lockfile',
'jsonschema',
'PyYAML',
'uritemplate', # used by hateoas navigator
'subprocess32; python_version<"3"',
'ipaddress; python_version<"3"', # used by whitelistfirewall
......
......@@ -27,15 +27,49 @@
#
##############################################################################
import argparse
import json
import os.path
import pprint
import lxml.etree
import six
import yaml
from slapos.cli.config import ClientConfigCommand
from slapos.client import init, ClientConfig, _getSoftwareReleaseFromSoftwareString
from slapos.client import (ClientConfig, _getSoftwareReleaseFromSoftwareString,
init)
from slapos.slap import ResourceNotReady
from slapos.util import SoftwareReleaseSchema, SoftwareReleaseSerialisation
try:
from typing import IO, Dict
except ImportError:
pass
def getParametersFromFile(file, serialisation):
# type: (IO[str], SoftwareReleaseSerialisation) -> Dict
extension = os.path.splitext(file.name)[1]
if extension in ('.yaml', '.yml'):
params = yaml.safe_load(file)
elif extension == '.xml':
tree = lxml.etree.parse(file)
params = {e.attrib['id']: e.text for e in tree.findall('/parameter')}
# because the use case of xml files is to copy paste existing XML parameters
# as found on slapos web interface, we aren't clever regarding the
# serialisation and assume they are already correct.
serialisation = None
else:
params = json.load(file)
if serialisation == SoftwareReleaseSerialisation.JsonInXml and list(params.keys()) != ['_']:
params = {'_': json.dumps(params)}
return params
def parse_option_dict(options):
# type: (str) -> Dict
"""
Parse a list of option strings like foo=bar baz=qux and return a dictionary.
Will raise if keys are repeated.
......@@ -78,10 +112,18 @@ class RequestCommand(ClientConfigCommand):
action='store_true',
help='Ask for a slave instance')
ap.add_argument('--parameters',
nargs='+',
help="Give your configuration 'option1=value1 option2=value2'")
parameter_args = ap.add_mutually_exclusive_group()
parameter_args.add_argument(
'--parameters',
nargs='+',
help="Instance parameters, in the form 'option1=value1 option2=value2'.")
parameter_args.add_argument(
'--parameters-file',
type=argparse.FileType('r'),
help="Instance parameters, in a file.\n"
"The file will be interpreted as json, yaml or xml depending on the file extension.")
return ap
def take_action(self, args):
......@@ -95,6 +137,16 @@ class RequestCommand(ClientConfigCommand):
do_request(self.app.log, conf, local)
# BBB on python3 we can use pprint.pformat
class StrPrettyPrinter(pprint.PrettyPrinter):
"""A PrettyPrinter which produces consistent output on python 2 and 3
"""
def format(self, object, context, maxlevels, level):
if six.PY2 and isinstance(object, six.text_type):
object = object.encode('utf-8')
return pprint.PrettyPrinter.format(self, object, context, maxlevels, level)
def do_request(logger, conf, local):
logger.info('Requesting %s as instance of %s...',
conf.reference, conf.software_url)
......@@ -104,11 +156,17 @@ def do_request(logger, conf, local):
if conf.software_url in local:
conf.software_url = local[conf.software_url]
software_schema = SoftwareReleaseSchema(conf.software_url, conf.type)
software_schema_serialisation = software_schema.getSerialisation()
parameters = conf.parameters
if conf.parameters_file:
parameters = getParametersFromFile(conf.parameters_file, software_schema_serialisation)
try:
partition = local['slap'].registerOpenOrder().request(
software_release=conf.software_url,
partition_reference=conf.reference,
partition_parameter_kw=conf.parameters,
partition_parameter_kw=parameters,
software_type=conf.type,
filter_kw=conf.node,
state=conf.state,
......@@ -116,7 +174,11 @@ def do_request(logger, conf, local):
)
logger.info('Instance requested.\nState is : %s.', partition.getState())
logger.info('Connection parameters of instance are:')
logger.info(pprint.pformat(partition.getConnectionParameterDict()))
connection_parameter_dict = partition.getConnectionParameterDict()
if software_schema_serialisation == SoftwareReleaseSerialisation.JsonInXml:
if '_' in connection_parameter_dict:
connection_parameter_dict = json.loads(connection_parameter_dict['_'])
logger.info(StrPrettyPrinter().pformat(connection_parameter_dict))
logger.info('You can rerun the command to get up-to-date information.')
except ResourceNotReady:
logger.warning('Instance requested. Master is provisioning it. Please rerun in a '
......
......@@ -40,14 +40,17 @@ import os
import logging
import re
from functools import wraps
import warnings
import json
import jsonschema
import six
from .exception import ResourceNotReady, ServerError, NotFoundError, \
ConnectionError
from .hateoas import SlapHateoasNavigator, ConnectionHelper
from slapos.util import loads, dumps, bytes2str, unicode2str, xml2dict, dict2xml, calculate_dict_hash
from slapos.util import (SoftwareReleaseSchema, bytes2str, calculate_dict_hash,
dict2xml, dumps, loads, unicode2str, xml2dict)
from xml.sax import saxutils
from zope.interface import implementer
......@@ -87,6 +90,27 @@ class SlapRequester(SlapDocument):
"""
def _requestComputerPartition(self, request_dict):
try:
SoftwareReleaseSchema(
request_dict['software_release'],
request_dict['software_type']
).validateInstanceParameterDict(
loads(request_dict['partition_parameter_xml']))
except jsonschema.ValidationError as e:
warnings.warn(
"Request parameters do not validate against schema definition:\n{e}".format(e=e),
UserWarning,
)
except Exception as e:
# note that we intentionally catch wide exceptions, so that if anything
# is wrong with fetching the schema or the schema itself this does not
# prevent users from requesting instances.
warnings.warn(
"Error validating request parameters against schema definition:\n{e.__class__.__name__} {e}".format(e=e),
UserWarning,
)
try:
xml = self._connection_helper.POST('requestComputerPartition', data=request_dict)
except ResourceNotReady:
......
......@@ -119,8 +119,10 @@ class ManagedHTTPServer(ManagedResource):
proto = 'http'
# hostname to listen to, default to ipv4 address of the current test
hostname = None # type: str
# port to listen to, default
# port to listen to, default to a free port selected with `findFreeTCPPort`
port = None # type: int
# current working directory of the server process
working_directory = None # type: str
@property
def url(self):
......@@ -166,10 +168,19 @@ class ManagedHTTPServer(ManagedResource):
if not self.port:
self.port = findFreeTCPPort(self.hostname)
def serve_forever(server, cwd):
if cwd:
os.chdir(cwd)
server.serve_forever()
server = self._makeServer()
self._process = multiprocessing.Process(
target=server.serve_forever,
target=serve_forever,
name=self._name,
kwargs={
'server': server,
'cwd': self.working_directory,
}
)
self._process.start()
# from now on, socket is used by server subprocess, we can close it
......
......@@ -25,6 +25,7 @@
#
##############################################################################
import json
import logging
import pprint
import unittest
......@@ -40,6 +41,7 @@ from contextlib import contextmanager
from mock import patch, create_autospec
import mock
from slapos.util import sqlite_connect, bytes2str
from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
import slapos.cli.console
import slapos.cli.entry
......@@ -49,10 +51,12 @@ import slapos.cli.computer_info
import slapos.cli.computer_list
import slapos.cli.computer_token
import slapos.cli.supervisorctl
import slapos.cli.request
from slapos.cli.proxy_show import do_show, StringIO
from slapos.cli.cache import do_lookup as cache_do_lookup
from slapos.cli.cache_source import do_lookup as cache_source_do_lookup
from slapos.client import ClientConfig
from slapos.slap import SoftwareProductCollection
import slapos.grid.svcbackend
import slapos.proxy
import slapos.slap
......@@ -65,8 +69,8 @@ def raiseNotFoundError(*args, **kwargs):
class CliMixin(unittest.TestCase):
def setUp(self):
slap = slapos.slap.slap()
self.local = {'slap': slap}
self.logger = create_autospec(logging.Logger)
self.local = {'slap': slap, 'product': SoftwareProductCollection(self.logger, slap)}
self.conf = create_autospec(ClientConfig)
class TestCliCache(CliMixin):
......@@ -641,3 +645,227 @@ class TestCliComplete(CliMixin):
with patch.object(sys, 'stdout', StringIO()) as app_stdout:
self.assertEqual(slapos.cli.entry.SlapOSApp().run(['complete', '--shell=fish']), 0)
self.assertIn('__fish_seen_subcommand_from', app_stdout.getvalue())
class TestCliRequest(CliMixin):
def test_parse_option_dict(self):
parse_option_dict = slapos.cli.request.parse_option_dict
self.assertEqual(parse_option_dict(['foo=bar', 'a=b']), {'foo': 'bar', 'a': 'b'})
# malformed option = assignment
self.assertRaises(ValueError, parse_option_dict, ['a'])
# duplicated key
self.assertRaises(ValueError, parse_option_dict, ['a=b', 'a=c'])
# corner cases
self.assertEqual(parse_option_dict(['a=a=b']), {'a': 'a=b'})
self.assertEqual(parse_option_dict(['a=a\nb']), {'a': 'a\nb'})
self.assertEqual(parse_option_dict([]), {})
def test_request(self):
self.conf.reference = 'instance reference'
self.conf.software_url = 'software URL'
self.conf.parameters = {'key': 'value'}
self.conf.parameters_file = None
self.conf.node = {'computer_guid': 'COMP-1234'}
self.conf.type = None
self.conf.state = None
self.conf.slave = False
with patch.object(
slapos.slap.slap,
'registerOpenOrder',
return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
slapos.cli.request.do_request(self.logger, self.conf, self.local)
registerOpenOrder().request.assert_called_once_with(
software_release='software URL',
partition_reference='instance reference',
partition_parameter_kw={'key': 'value'},
software_type=None,
filter_kw={'computer_guid': 'COMP-1234'},
state=None,
shared=False,
)
self.logger.info.assert_any_call(
'Requesting %s as instance of %s...',
'instance reference',
'software URL',
)
def test_request_json_in_xml_published_parameters(self):
tmpdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, tmpdir)
with open(os.path.join(tmpdir, 'software.cfg.json'), 'w') as f:
json.dump(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
DEFAULT_SOFTWARE_TYPE: {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
}, f)
self.conf.reference = 'instance reference'
self.conf.software_url = os.path.join(tmpdir, 'software.cfg')
self.conf.parameters = {'key': 'value'}
self.conf.parameters_file = None
self.conf.node = {'computer_guid': 'COMP-1234'}
self.conf.type = None
self.conf.state = None
self.conf.slave = False
cp = slapos.slap.ComputerPartition(
'computer_%s' % self.id(),
'partition_%s' % self.id())
cp._requested_state = 'started'
cp._connection_dict = {'_': json.dumps({'foo': 'bar'})}
with patch.object(
slapos.slap.slap,
'registerOpenOrder',
return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
registerOpenOrder().request.return_value = cp
slapos.cli.request.do_request(self.logger, self.conf, self.local)
registerOpenOrder().request.assert_called_once()
self.assertEqual(self.logger.info.mock_calls, [
mock.call('Requesting %s as instance of %s...', self.conf.reference,
self.conf.software_url),
mock.call('Instance requested.\nState is : %s.', 'started'),
mock.call('Connection parameters of instance are:'),
mock.call("{'foo': 'bar'}"),
mock.call('You can rerun the command to get up-to-date information.'),
])
class TestCliRequestParametersFileJson(CliMixin):
"""Request with --parameter-file, with a .json file.
"""
expected_partition_parameter_kw = {'foo': ['bar']}
def _makeParameterFile(self):
f = tempfile.NamedTemporaryFile(suffix='.json', mode='w', delete=False)
self.addCleanup(os.unlink, f.name)
f.write(textwrap.dedent('''\
{
"foo": ["bar"]
}
'''))
f.flush()
return f.name
def test_request_parameters_file(self):
self.conf.reference = 'instance reference'
self.conf.software_url = 'software URL'
self.conf.parameters = None
f = open(self._makeParameterFile())
self.addCleanup(f.close)
self.conf.parameters_file = f
self.conf.node = {'computer_guid': 'COMP-1234'}
self.conf.type = None
self.conf.state = None
self.conf.slave = False
with patch.object(
slapos.slap.slap,
'registerOpenOrder',
return_value=mock.create_autospec(slapos.slap.OpenOrder)) as registerOpenOrder:
slapos.cli.request.do_request(self.logger, self.conf, self.local)
registerOpenOrder().request.assert_called_once_with(
software_release='software URL',
partition_reference='instance reference',
partition_parameter_kw=self.expected_partition_parameter_kw,
software_type=None,
filter_kw={'computer_guid': 'COMP-1234'},
state=None,
shared=False,
)
self.logger.info.assert_any_call(
'Requesting %s as instance of %s...',
'instance reference',
'software URL',
)
class TestCliRequestParametersFileJsonJsonInXMLSerialisation(
TestCliRequestParametersFileJson):
"""Request with --parameter-file, with a .json file and a software using
json-in-xml for serialisation. In that case, the parameters are automatically
serialised with {'_': json.dumps(params)}
"""
expected_partition_parameter_kw = {"_": "{\"foo\": [\"bar\"]}"}
def test_request_parameters_file(self):
with mock.patch(
'slapos.cli.request.SoftwareReleaseSchema.getSerialisation',
return_value='json-in-xml'):
super(TestCliRequestParametersFileJsonJsonInXMLSerialisation,
self).test_request_parameters_file()
class TestCliRequestParametersFileJsonJsonInXMLSerialisationAlreadySerialised(
TestCliRequestParametersFileJson):
"""Request with --parameter-file, with a .json file and a software using
json-in-xml for serialisation and parameters already serialised with
{'_': json.dumps(params)}. In that case, parameters are not serialized one
more time.
"""
expected_partition_parameter_kw = {"_": "{\"foo\": [\"bar\"]}"}
def _makeParameterFile(self):
f = tempfile.NamedTemporaryFile(suffix='.json', mode='w', delete=False)
self.addCleanup(os.unlink, f.name)
f.write(textwrap.dedent(r'''
{"_": "{\"foo\": [\"bar\"]}"}
'''))
f.flush()
return f.name
def test_request_parameters_file(self):
with mock.patch(
'slapos.cli.request.SoftwareReleaseSchema.getSerialisation',
return_value='json-in-xml'):
super(
TestCliRequestParametersFileJsonJsonInXMLSerialisationAlreadySerialised,
self).test_request_parameters_file()
class TestCliRequestParametersFileYaml(TestCliRequestParametersFileJson):
"""Request with --parameter-file, with a .yaml file. This behaves like json.
"""
def _makeParameterFile(self):
f = tempfile.NamedTemporaryFile(suffix='.yaml', mode='w', delete=False)
self.addCleanup(os.unlink, f.name)
f.write(textwrap.dedent('''\
foo:
- bar
'''))
f.flush()
return f.name
class TestCliRequestParametersFileXml(TestCliRequestParametersFileJson):
"""Request with --parameter-file, with a .xml file
"""
expected_partition_parameter_kw = {'foo': 'bar'}
def _makeParameterFile(self):
f = tempfile.NamedTemporaryFile(suffix='.xml', mode='w', delete=False)
f.write(textwrap.dedent('''\
<?xml version="1.0" encoding="utf-8"?>
<instance>
<parameter id="foo">bar</parameter>
</instance>
'''))
f.flush()
self.addCleanup(os.unlink, f.name)
return f.name
......@@ -32,6 +32,7 @@ from six.moves.urllib import parse
from six import PY3
import tempfile
import logging
import warnings
from collections import OrderedDict
import httmock
......@@ -872,6 +873,188 @@ class TestComputerPartition(SlapMixin):
content_list = f.read().splitlines()
self.assertEqual(sorted(content_list), ['myref', 'mysecondref'])
def test_request_validate_request_parameter(self):
def handler(url, req):
if url.path.endswith('/software.cfg.json'):
return json.dumps(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
'default': {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
})
if url.path.endswith('/instance-default-input-schema.json'):
return json.dumps(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Simple instance parameters schema for tests",
"required": ["foo"],
"properties": {
"foo": {
"$ref": "./schemas-definitions.json#/foo"
}
},
"type": "object"
})
if url.path.endswith('/schemas-definitions.json'):
return json.dumps({"foo": {"type": "string", "const": "bar"}})
raise ValueError(404)
with httmock.HTTMock(handler):
with mock.patch.object(warnings, 'warn') as warn:
cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
cp._connection_helper = mock.Mock()
cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
cp.request(
'https://example.com/software.cfg', 'default', 'reference',
partition_parameter_kw={'foo': 'bar'})
warn.assert_not_called()
with httmock.HTTMock(handler):
with mock.patch.object(warnings, 'warn') as warn:
cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
cp._connection_helper = mock.Mock()
cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
cp.request(
'https://example.com/software.cfg', 'default', 'reference',
partition_parameter_kw={'foo': 'baz'})
if PY3:
warn.assert_called_with(
"Request parameters do not validate against schema definition:\n"
"'bar' was expected\n\n"
"Failed validating 'const' in schema['properties']['foo']:\n"
" {'const': 'bar', 'type': 'string'}\n\n"
"On instance['foo']:\n 'baz'", UserWarning
)
else: # BBB
warn.assert_called_with(
"Request parameters do not validate against schema definition:\n"
"u'bar' was expected\n\n"
"Failed validating u'const' in schema[u'properties'][u'foo']:\n"
" {u'const': u'bar', u'type': u'string'}\n\n"
"On instance[u'foo']:\n 'baz'", UserWarning
)
def test_request_validate_request_parameter_broken_software_release_schema(self):
"""Corner case tests for incorrect software release schema, these should
not prevent the request (mostly for backward compatibility)
"""
def wrong_software_cfg_schema(url, req):
if url.path.endswith('/software.cfg.json'):
return "wrong"
raise ValueError(404)
def wrong_instance_parameter_schema(url, req):
if url.path.endswith('/software.cfg.json'):
return json.dumps(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
'default': {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
})
if url.path.endswith('/instance-default-input-schema.json'):
return "wrong"
raise ValueError(404)
def invalid_instance_parameter_schema(url, req):
if url.path.endswith('/software.cfg.json'):
return json.dumps(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
'default': {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
})
if url.path.endswith('/instance-default-input-schema.json'):
return json.dumps(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Invalid json schema",
"required": {"wrong": True},
"properties": {
["wrong schema"]
},
"type": "object"
})
raise ValueError(404)
def broken_reference(url, req):
if url.path.endswith('/software.cfg.json'):
return json.dumps(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": "json-in-xml",
"software-type": {
'default': {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
}
})
if url.path.endswith('/instance-default-input-schema.json'):
return json.dumps(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Simple instance parameters schema for tests",
"required": ["foo"],
"properties": {
"foo": {
"$ref": "broken"
}
},
"type": "object"
})
raise ValueError(404)
for handler, warning_expected in (
(broken_reference, True),
(wrong_software_cfg_schema, False),
(wrong_instance_parameter_schema, True),
(invalid_instance_parameter_schema, True),
):
with httmock.HTTMock(handler):
with mock.patch.object(warnings, 'warn') as warn:
cp = slapos.slap.ComputerPartition('computer_id', 'partition_id')
cp._connection_helper = mock.Mock()
cp._connection_helper.POST.side_effect = slapos.slap.ResourceNotReady
cp.request(
'https://example.com/software.cfg', 'default', 'reference',
partition_parameter_kw={'foo': 'bar'})
if warning_expected:
warn.assert_called()
else:
warn.assert_not_called()
def _test_new_computer_partition_state(self, state):
"""
Helper method to automate assertions of failing states on new Computer
......
......@@ -24,14 +24,26 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import functools
import json
import logging
import os
import slapos.util
from slapos.util import string_to_boolean, unicode2str
import shutil
import tempfile
import textwrap
import unittest
import shutil
from pwd import getpwnam
from six.moves import SimpleHTTPServer
import jsonschema
import slapos.util
from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
from slapos.testing.utils import ManagedHTTPServer
from slapos.util import (SoftwareReleaseSchema, SoftwareReleaseSerialisation,
string_to_boolean, unicode2str)
class TestUtil(unittest.TestCase):
"""
Tests methods available in the slapos.util module.
......@@ -232,5 +244,221 @@ class TestUtil(unittest.TestCase):
self.assertRaises(Exception, slapos.util.dumps, Nasty())
class SoftwareReleaseSchemaTestXmlSerialisationMixin:
serialisation = SoftwareReleaseSerialisation.Xml
class SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin:
serialisation = SoftwareReleaseSerialisation.JsonInXml
class SoftwareReleaseSchemaTestMixin(object):
"""Mixin with test methods
"""
software_url = None # type: str
serialisation = None # type: SoftwareReleaseSerialisation
def test_software_schema(self):
schema = SoftwareReleaseSchema(self.software_url, None)
software_schema = schema.getSoftwareSchema()
self.assertEqual(software_schema['name'], 'Test Software')
self.assertEqual(len(software_schema['software-type']), 2)
def test_serialisation(self):
schema = SoftwareReleaseSchema(self.software_url, None)
self.assertEqual(schema.getSerialisation(), self.serialisation)
def test_instance_request_parameter_schema_default_software_type(self):
schema = SoftwareReleaseSchema(self.software_url, None)
self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
instance_parameter_schema = schema.getInstanceRequestParameterSchema()
self.assertEqual(instance_parameter_schema['description'],
"Simple instance parameters schema for tests")
def test_connection_parameter_schema(self):
schema = SoftwareReleaseSchema(self.software_url, None)
self.assertTrue(schema.getInstanceConnectionParameterSchemaURL())
instance_parameter_schema = schema.getInstanceConnectionParameterSchema()
self.assertEqual(instance_parameter_schema['description'],
"Simple connection parameters schema for tests")
def test_instance_request_parameter_validate_default_software_type(self):
schema = SoftwareReleaseSchema(self.software_url, None)
self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
instance_ok = {'key': 'value', 'type': 'default'}
schema.validateInstanceParameterDict(instance_ok)
if self.serialisation == SoftwareReleaseSerialisation.JsonInXml:
# already serialized values are also tolerated
schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict({"wrong": True})
instance_ok['key'] = False # wrong type
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict(instance_ok)
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
def test_instance_request_parameter_validate_alternate_software_type(self):
schema = SoftwareReleaseSchema(self.software_url, 'alternate')
self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
instance_ok = {'key': 'value', 'type': 'alternate'}
schema.validateInstanceParameterDict(instance_ok)
if self.serialisation == SoftwareReleaseSerialisation.JsonInXml:
# already serialized values are also tolerated
schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict({"wrong": True})
instance_ok['type'] = 'wrong'
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict(instance_ok)
with self.assertRaises(jsonschema.ValidationError):
schema.validateInstanceParameterDict({'_': json.dumps(instance_ok)})
def test_instance_request_parameter_schema_alternate_software_type(self):
schema = SoftwareReleaseSchema(self.software_url, 'alternate')
self.assertTrue(schema.getInstanceRequestParameterSchemaURL())
instance_parameter_schema = schema.getInstanceRequestParameterSchema()
self.assertEqual(instance_parameter_schema['description'],
"Simple instance parameters schema for tests")
class SoftwareReleaseSchemaTestFileSoftwareReleaseMixin(SoftwareReleaseSchemaTestMixin):
"""Mixin with tests and software release profiles and schema in a
temporary directory.
"""
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.addCleanup(shutil.rmtree, self.tmpdir)
tmpfile = functools.partial(os.path.join, self.tmpdir)
with open(tmpfile('software.cfg'), 'w') as f:
f.write(
textwrap.dedent("""\
[buildout]
"""))
with open(tmpfile('software.cfg.json'), 'w') as f:
json.dump(
{
"name": "Test Software",
"description": "Dummy software for Test",
"serialisation": self.serialisation,
"software-type": {
DEFAULT_SOFTWARE_TYPE: {
"title": "Default",
"description": "Default type",
"request": "instance-default-input-schema.json",
"response": "instance-default-output-schema.json",
"index": 0
},
"alternate": {
"title": "Alternate",
"description": "Alternate type",
"request": "instance-alternate-input-schema.json",
"response": "instance-alternate-output-schema.json",
"index": 0
},
}
}, f)
for software_type in ('default', 'alternate'):
with open(
tmpfile('instance-{software_type}-input-schema.json'.format(
software_type=software_type)), 'w') as f:
json.dump(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Simple instance parameters schema for tests",
"required": ["key", "type"],
"properties": {
"key": {
"$ref": "./schemas-definitions.json#/key"
},
"type": {
"type": "string",
"const": software_type
}
},
"type": "object"
}, f)
with open(
tmpfile('instance-{software_type}-output-schema.json'.format(
software_type=software_type)), 'w') as f:
json.dump(
{
"$schema": "http://json-schema.org/draft-07/schema",
"description": "Simple connection parameters schema for tests",
}, f)
with open(tmpfile('schemas-definitions.json'), 'w') as f:
json.dump({"key": {"type": "string"}}, f)
self.software_url = tmpfile('software.cfg')
class SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin(SoftwareReleaseSchemaTestFileSoftwareReleaseMixin):
"""Mixin serving software release files over http.
"""
def setUp(self):
super(SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin, self).setUp()
class ProfileHTTPServer(ManagedHTTPServer):
hostname = os.environ['SLAPOS_TEST_IPV4']
working_directory = self.tmpdir
RequestHandler = SimpleHTTPServer.SimpleHTTPRequestHandler
self.logger = logging.getLogger(self.id())
self.logger.propagate = False
server = ProfileHTTPServer(self, 'server')
server.open()
self.addCleanup(server.close)
self.software_url = server.url + '/software.cfg'
class TestSoftwareReleaseSchemaFileSoftwareReleaseXmlSerialisation(
SoftwareReleaseSchemaTestXmlSerialisationMixin,
SoftwareReleaseSchemaTestFileSoftwareReleaseMixin,
unittest.TestCase):
pass
class TestSoftwareReleaseSchemaFileSoftwareReleaseJsonInXmlSerialisation(
SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin,
SoftwareReleaseSchemaTestFileSoftwareReleaseMixin,
unittest.TestCase):
pass
class TestSoftwareReleaseSchemaHTTPSoftwareReleaseXmlSerialisation(
SoftwareReleaseSchemaTestXmlSerialisationMixin,
SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin,
unittest.TestCase):
pass
class TestSoftwareReleaseSchemaHTTPSoftwareReleaseJsonInXmlSerialisation(
SoftwareReleaseSchemaTestJsonInXmlSerialisationMixin,
SoftwareReleaseSchemaTestHTTPSoftwareReleaseMixin,
unittest.TestCase):
pass
class TestSoftwareReleaseSchemaEdgeCases(unittest.TestCase):
def test_software_schema_file_not_exist(self):
schema = SoftwareReleaseSchema('/file/not/exist', None)
self.assertIsNone(schema.getSoftwareSchema())
def test_software_schema_wrong_URL(self):
schema = SoftwareReleaseSchema('http://slapos.invalid/software.cfg', None)
self.assertIsNone(schema.getSoftwareSchema())
if __name__ == '__main__':
unittest.main()
......@@ -27,19 +27,32 @@
#
##############################################################################
import enum
import errno
import hashlib
import json
import os
import shutil
import socket
import sqlite3
import struct
import subprocess
import sqlite3
from xml_marshaller.xml_marshaller import Marshaller, Unmarshaller
from lxml import etree
import warnings
import jsonschema
import netaddr
import requests
import six
from lxml import etree
from six.moves.urllib import parse
import hashlib
import netaddr
import shutil
from six.moves.urllib_parse import urljoin
from xml_marshaller.xml_marshaller import Marshaller, Unmarshaller
try:
from typing import Dict, Optional, IO
except ImportError:
pass
try:
......@@ -307,3 +320,150 @@ def rmtree(path):
raise e # XXX make pylint happy
shutil.rmtree(path, onerror=chmod_retry)
def _readAsJson(url):
# type: (str) -> Optional[Dict]
"""Reads and parse the json file located at `url`.
`url` can also be the path of a local file.
"""
if url.startswith('file://'):
url = url[len('file://'):]
path = url if os.path.exists(url) else None
if path:
with open(path) as f:
try:
return json.load(f)
except ValueError:
return None
if url.startswith('http://') or url.startswith('https://'):
try:
r = requests.get(url)
r.raise_for_status()
return r.json()
except (requests.exceptions.RequestException, ValueError):
return None
return None
class SoftwareReleaseSerialisation(str, enum.Enum):
Xml = 'xml'
JsonInXml = 'json-in-xml'
class SoftwareReleaseSchema(object):
def __init__(self, software_url, software_type):
# type: (str, Optional[str]) -> None
self.software_url = software_url
self.software_type = software_type
def getSoftwareSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema for this software.
"""
return _readAsJson(self.software_url + '.json')
def getSoftwareTypeSchema(self):
# type: () -> Optional[Dict]
"""Returns schema for this software type.
"""
software_schema = self.getSoftwareSchema()
if software_schema is None:
return None
software_type = self.software_type
from slapos.slap.slap import DEFAULT_SOFTWARE_TYPE
if software_type is None:
software_type = DEFAULT_SOFTWARE_TYPE
# XXX Some software are using "default" for default software type, because
# we are transitionning from DEFAULT_SOFTWARE_TYPE ("RootSoftwareInstance")
# to "default".
if software_type == DEFAULT_SOFTWARE_TYPE \
and software_type not in software_schema['software-type'] \
and 'default' in software_schema['software-type']:
warnings.warn(
"Software release {} does not have schema for DEFAULT_SOFTWARE_TYPE but has one for 'default'."
" Using 'default' instead.".format(self.software_url),
UserWarning,
)
software_type = 'default'
return software_schema['software-type'].get(software_type)
def getSerialisation(self):
# type: () -> Optional[SoftwareReleaseSerialisation]
"""Returns the serialisation method used for parameters.
"""
software_schema = self.getSoftwareSchema()
if software_schema is None:
return None
return SoftwareReleaseSerialisation(software_schema['serialisation'])
def getInstanceRequestParameterSchemaURL(self):
# type: () -> Optional[str]
"""Returns the URL of the schema defining instance parameters.
"""
software_type_schema = self.getSoftwareTypeSchema()
if software_type_schema is None:
return None
software_url = self.software_url
if os.path.exists(software_url):
software_url = 'file://' + software_url
return urljoin(software_url, software_type_schema['request'])
def getInstanceRequestParameterSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema defining instance parameters.
"""
instance_parameter_schema_url = self.getInstanceRequestParameterSchemaURL()
if instance_parameter_schema_url is None:
return None
schema = _readAsJson(instance_parameter_schema_url)
if schema:
# so that jsonschema knows how to resolve references
schema.setdefault('$id', instance_parameter_schema_url)
return schema
def getInstanceConnectionParameterSchemaURL(self):
# type: () -> Optional[str]
"""Returns the URL of the schema defining connection parameters published by the instance.
"""
software_type_schema = self.getSoftwareTypeSchema()
if software_type_schema is None:
return None
return urljoin(self.software_url, software_type_schema['response'])
def getInstanceConnectionParameterSchema(self):
# type: () -> Optional[Dict]
"""Returns the schema defining connection parameters published by the instance.
"""
instance_parameter_schema_url = self.getInstanceConnectionParameterSchemaURL()
if instance_parameter_schema_url is None:
return None
schema = _readAsJson(instance_parameter_schema_url)
if schema:
# so that jsonschema knows how to resolve references
schema.setdefault('$id', instance_parameter_schema_url)
return schema
def validateInstanceParameterDict(self, parameter_dict):
# type: (Dict) -> None
"""Validate instance parameters against the software schema.
Raise jsonschema.ValidationError if parameters does not validate.
"""
schema_url = self.getInstanceRequestParameterSchemaURL()
if schema_url:
instance = parameter_dict
if self.getSerialisation() == SoftwareReleaseSerialisation.JsonInXml:
try:
instance = json.loads(parameter_dict['_'])
except KeyError:
instance = parameter_dict
instance.pop('$schema', None)
jsonschema.validate(
instance=instance,
schema=self.getInstanceRequestParameterSchema(),
)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment