Commit 655fb891 authored by Xavier Thompson's avatar Xavier Thompson

slapproxy: Add minimal hateoas support for cli

Add the minimum hateoas support required for the cli commands:
- `slapos service list`
- `slapos service info <reference>`
- `slapos computer list`
- `slapos computer info <reference>`

To enable hateoas, complete the slapos.cfg configuration with:
```
[slapos]
master_rest_url = <proxyaddr>/hateoas
```

Also: Fix some bugs and python3 incompatibilities in slap/hateoas.py.

See merge request nexedi/slapos.core!290
parent 740021b4
...@@ -31,6 +31,7 @@ ...@@ -31,6 +31,7 @@
import random import random
import string import string
import time import time
import re
from datetime import datetime from datetime import datetime
from slapos.slap.slap import Computer, ComputerPartition, \ from slapos.slap.slap import Computer, ComputerPartition, \
SoftwareRelease, SoftwareInstance, NotFoundError SoftwareRelease, SoftwareInstance, NotFoundError
...@@ -39,12 +40,12 @@ import slapos.slap ...@@ -39,12 +40,12 @@ import slapos.slap
from slapos.util import bytes2str, unicode2str, sqlite_connect, \ from slapos.util import bytes2str, unicode2str, sqlite_connect, \
xml2dict, dict2xml xml2dict, dict2xml
from flask import g, Flask, request, abort from flask import g, Flask, request, abort, redirect, url_for
from slapos.util import loads, dumps from slapos.util import loads, dumps
import six import six
from six.moves import range from six.moves import range
from six.moves.urllib.parse import urlparse from six.moves.urllib.parse import urlparse, unquote, urljoin
app = Flask(__name__) app = Flask(__name__)
...@@ -930,3 +931,186 @@ def getSoftwareReleaseListFromSoftwareProduct(): ...@@ -930,3 +931,186 @@ def getSoftwareReleaseListFromSoftwareProduct():
software_release_url_list = [] software_release_url_list = []
return dumps(software_release_url_list) return dumps(software_release_url_list)
# hateoas routing
# ---------------
# We only need to handle the hateoas requests made by
# slapos service list
# slapos service info <reference>
# slapos computer list
# slapos computer info <reference>
def unquoted_url_for(method, **kwargs):
return unquote(url_for(method, **kwargs))
def busy_root_partitions_list(title=None):
partitions = []
query = 'SELECT * FROM %s WHERE slap_state<>"free" AND requested_by IS NULL'
args = []
if title:
query += ' AND partition_reference=?'
args.append(title)
for row in execute_db('partition', query, args):
p = dict(row)
p['url_string'] = p['software_release']
p['title'] = p['partition_reference']
p['relative_url'] = url_for('hateoas_partitions', partition_reference=p['partition_reference'])
partitions.append(p)
return partitions
def computers_list(reference=None):
computers = []
query = 'SELECT * FROM %s'
args = []
if reference:
query += ' WHERE reference=?'
args.append(reference)
for row in execute_db('computer', query, args):
c = dict(row)
c['title'] = c['reference']
c['relative_url'] = url_for('hateoas_computers', computer_reference=c['reference'])
computers.append(c)
return computers
r_string = re.compile('"((\\.|[^\\"])*)"')
def is_valid(name):
match = r_string.match(name)
if match.group(0) == name:
return True
return False
p_service_list = 'portal_type:"Hosting Subscription" AND validation_state:validated'
p_service_info = p_service_list + ' AND title:='
p_computer_list = 'portal_type:"Computer" AND validation_state:validated'
p_computer_info = p_computer_list + ' AND reference:='
def parse_query(query):
if query == p_service_list:
return busy_root_partitions_list()
elif query.startswith(p_service_info):
title = query[len(p_service_info):]
if is_valid(title):
return busy_root_partitions_list(title.strip('"'))
elif query == p_computer_list:
return computers_list()
elif query.startswith(p_computer_info):
reference = query[len(p_computer_info):]
if is_valid(reference):
return computers_list(reference.strip('"'))
return None
@app.route('/hateoas/partitions/<partition_reference>', methods=['GET'])
def hateoas_partitions(partition_reference):
partition = execute_db('partition', 'SELECT * FROM %s WHERE partition_reference=?', [partition_reference], one=True)
if partition is None:
abort(404)
return {
'_embedded': {
'_view': {
'form_id': {
'type': 'StringField',
'key': 'partition',
'default': partition['reference'],
},
'my_reference': {
'type': 'StringField',
'key': 'partition_reference',
'default': partition['partition_reference'],
},
'my_slap_state': {
'type': 'StringField',
'key': 'slap_state',
'default': partition['slap_state'],
},
'my_text_content': {
'type': 'StringField',
'key': 'xml',
'default': partition['xml'],
},
'my_connection_parameter_list': {
'type': 'StringField',
'key': 'connection_xml',
'default': partition['connection_xml'],
},
'my_url_string': {
'type': 'StringField',
'key': 'software_release',
'default': partition['software_release'],
},
},
},
'_links': {
'type': {
'name': 'Hosting Subscription',
},
},
}
@app.route('/hateoas/computers/<computer_reference>', methods=['GET'])
def hateoas_computers(computer_reference):
computer = execute_db('computer', 'SELECT * FROM %s WHERE reference=?', [computer_reference], one=True)
if computer is None:
abort(404)
return {
'_embedded': {
'_view': {
'form_id': {
'type': 'StringField',
'key': 'computer',
'default': computer['reference'],
},
'my_reference': {
'type': 'StringField',
'key': 'reference',
'default': computer['reference'],
},
'my_title': {
'type': 'StringField',
'key': 'reference',
'default': computer['reference'],
},
},
},
'_links': {
'type': {
'name': 'Computer',
},
},
}
def hateoas_traverse():
return redirect(request.args.get('relative_url'))
def hateoas_search():
contents = parse_query(request.args.get("query"))
if contents is None:
abort(400)
return { '_embedded': {'contents': contents} }
def hateoas_root():
return {
'_links': {
'raw_search': {
'href': urljoin(request.host_url, unquoted_url_for('hateoas', mode='search', query='{query}', select_list='{select_list}'))
},
'traverse': {
'href': urljoin(request.host_url, unquoted_url_for('hateoas', mode='traverse', relative_url='{relative_url}', view='{view}'))
},
}
}
mode_handlers = {
None: hateoas_root,
'search': hateoas_search,
'traverse': hateoas_traverse,
}
@app.route('/hateoas', methods=['GET'])
def hateoas():
mode = request.args.get('mode')
handler = mode_handlers.get(mode, lambda: abort(400))
resp = handler()
return resp
...@@ -383,7 +383,7 @@ class SlapHateoasNavigator(HateoasNavigator): ...@@ -383,7 +383,7 @@ class SlapHateoasNavigator(HateoasNavigator):
hosting_subscription_dict = {} hosting_subscription_dict = {}
for hosting_subscription in hosting_subscription_list: for hosting_subscription in hosting_subscription_list:
software_instance = TempDocument() software_instance = TempDocument()
for key, value in hosting_subscription.iteritems(): for key, value in six.iteritems(hosting_subscription):
if key in ['_links', 'url_string']: if key in ['_links', 'url_string']:
continue continue
setattr(software_instance, '_%s' % key, value) setattr(software_instance, '_%s' % key, value)
...@@ -397,7 +397,7 @@ class SlapHateoasNavigator(HateoasNavigator): ...@@ -397,7 +397,7 @@ class SlapHateoasNavigator(HateoasNavigator):
computer_dict = {} computer_dict = {}
for computer_json in computer_list: for computer_json in computer_list:
computer = TempDocument() computer = TempDocument()
for key, value in computer_json.iteritems(): for key, value in six.iteritems(computer_json):
if key in ['_links']: if key in ['_links']:
continue continue
setattr(computer, '_%s' % key, value) setattr(computer, '_%s' % key, value)
...@@ -420,6 +420,7 @@ class SlapHateoasNavigator(HateoasNavigator): ...@@ -420,6 +420,7 @@ class SlapHateoasNavigator(HateoasNavigator):
assert len(hosting_subscription_list) <= 1, \ assert len(hosting_subscription_list) <= 1, \
"There are more them one Hosting Subscription for this reference" "There are more them one Hosting Subscription for this reference"
hosting_subscription_jio_key= None
for hosting_subscription_candidate in hosting_subscription_list: for hosting_subscription_candidate in hosting_subscription_list:
if hosting_subscription_candidate.get('title') == reference: if hosting_subscription_candidate.get('title') == reference:
hosting_subscription_jio_key = hosting_subscription_candidate['relative_url'] hosting_subscription_jio_key = hosting_subscription_candidate['relative_url']
...@@ -437,6 +438,7 @@ class SlapHateoasNavigator(HateoasNavigator): ...@@ -437,6 +438,7 @@ class SlapHateoasNavigator(HateoasNavigator):
assert len(computer_list) <= 1, \ assert len(computer_list) <= 1, \
"There are more them one Computer for this reference" "There are more them one Computer for this reference"
computer_jio_key = None
for computer_candidate in computer_list: for computer_candidate in computer_list:
if computer_candidate.get("reference") == reference: if computer_candidate.get("reference") == reference:
computer_jio_key = computer_candidate['relative_url'] computer_jio_key = computer_candidate['relative_url']
......
...@@ -42,8 +42,12 @@ import sys ...@@ -42,8 +42,12 @@ import sys
import tempfile import tempfile
import time import time
import unittest import unittest
import json
import mock import mock
import requests
from six.moves.urllib.parse import urljoin
import slapos.proxy import slapos.proxy
import slapos.proxy.views as views import slapos.proxy.views as views
import slapos.slap import slapos.slap
...@@ -1163,6 +1167,185 @@ class TestSlaveRequest(MasterMixin): ...@@ -1163,6 +1167,185 @@ class TestSlaveRequest(MasterMixin):
self.assertEqual(slave._partition_id, partition._partition_id) self.assertEqual(slave._partition_id, partition._partition_id)
class TestAppSession(requests.Session):
"""
A request session that exposes the necessary interface to seamlessly
replace the object returned by views.app.test_client().
"""
def __init__(self, prefix_url, *args, **kwargs):
super(TestAppSession, self).__init__(*args, **kwargs)
self.prefix_url = prefix_url
def request(self, method, url, *args, **kwargs):
url = urljoin(self.prefix_url, url)
resp = super(TestAppSession, self).request(method, url, *args, **kwargs)
setattr(resp, '_status_code', resp.status_code)
setattr(resp, 'data', resp.content)
return resp
class CliMasterMixin(MasterMixin):
"""
Start a real proxy via the cli so that it will anwser to cli requests.
"""
def createSlapOSConfigurationFile(self):
host = os.environ['SLAPOS_TEST_IPV4']
self.proxyaddr = 'http://%s:8080' % host
with open(self.slapos_cfg, 'w') as f:
f.write("""[slapos]
software_root = %(tempdir)s/opt/slapgrid
instance_root = %(tempdir)s/srv/slapgrid
master_url = %(proxyaddr)s
master_rest_url = %(proxyaddr)s/hateoas
computer_id = computer
[slapproxy]
host = %(host)s
port = 8080
database_uri = %(tempdir)s/lib/proxy.db
""" % {'tempdir': self._tempdir, 'proxyaddr': self.proxyaddr, 'host': host})
def cliDoSlapos(self, command, method=subprocess.check_output, **kwargs):
return method(
(sys.executable, '-m', 'slapos.cli.entry') + command + ('--cfg', self.slapos_cfg),
env={"PYTHONPATH": ':'.join(sys.path)},
cwd=os.chdir(os.path.join(os.path.dirname(slapos.proxy.__file__), os.pardir, os.pardir)),
universal_newlines=True,
**kwargs
)
def startProxy(self):
self.proxy_process = self.cliDoSlapos(
('proxy', 'start'),
method=subprocess.Popen,
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
self.app = TestAppSession(self.proxyaddr)
# Wait a bit for proxy to be started
for attempts in range(1, 20):
try:
self.app.get('/')
except requests.ConnectionError:
time.sleep(0.1 * attempts)
else:
break
else:
self.fail('Could not start proxy.')
def tearDown(self):
self.proxy_process.kill()
self.proxy_process.wait()
super(CliMasterMixin, self).tearDown()
class TestCliInformation(CliMasterMixin):
"""
Test minimal hateoas support for cli.
"""
def test_computer_list(self):
self.format_for_number_of_partitions(1)
self.format_for_number_of_partitions(1, 'COMP-1')
self.format_for_number_of_partitions(1, 'COMP-2')
output = self.cliDoSlapos(('computer', 'list'), stderr=subprocess.DEVNULL).splitlines()
self.assertEqual(len(output), 4)
self.assertEqual(output[0], 'List of Computers:')
self.assertEqual(
sorted(output[1:]),
['COMP-1 COMP-1', 'COMP-2 COMP-2', 'computer computer'],
)
def test_computer_info(self):
self.format_for_number_of_partitions(1)
self.format_for_number_of_partitions(1, 'COMP-1')
self.format_for_number_of_partitions(1, 'COMP-2')
output0 = self.cliDoSlapos(('computer', 'info', 'computer'), stderr=subprocess.DEVNULL)
self.assertEqual(
output0.splitlines(),
['Computer Reference: computer', 'Computer Title : computer'],
)
output1 = self.cliDoSlapos(('computer', 'info', 'COMP-1'), stderr=subprocess.DEVNULL)
self.assertEqual(
output1.splitlines(),
['Computer Reference: COMP-1', 'Computer Title : COMP-1'],
)
output2 = self.cliDoSlapos(('computer', 'info', 'COMP-2'), stderr=subprocess.DEVNULL)
self.assertEqual(
output2.splitlines(),
['Computer Reference: COMP-2', 'Computer Title : COMP-2'],
)
def test_service_list(self):
self.format_for_number_of_partitions(4)
self.request('http://sr0//', None, 'MyInstance0', None)
self.request('http://sr1//', None, 'MyInstance1', None)
self.request('http://sr2//', None, 'MyInstance2', None)
self.request('http://sr3//', None, 'MyInstance3', 'slappart0')
output = self.cliDoSlapos(('service', 'list'), stderr=subprocess.DEVNULL).splitlines()
self.assertEqual(len(output), 4)
self.assertEqual(output[0], 'List of services:')
self.assertEqual(
sorted(output[1:]),
['MyInstance0 http://sr0//', 'MyInstance1 http://sr1//', 'MyInstance2 http://sr2//'],
)
def test_service_info(self):
self.format_for_number_of_partitions(3)
self.request('http://sr0//', None, 'MyInstance0', None)
self.request('http://sr1//', None, 'MyInstance1', None, partition_parameter_kw={'couscous': 'hello'})
self.request('http://sr2//', None, 'MyInstance2', 'slappart0')
output0 = self.cliDoSlapos(('service', 'info', 'MyInstance0'), stderr=subprocess.DEVNULL)
self.assertEqual(
output0.splitlines(),
[
'Software Release URL: http://sr0//',
'Instance state: busy',
'Instance parameters:',
'{}',
'Connection parameters:',
'None'
],
)
output1 = self.cliDoSlapos(('service', 'info', 'MyInstance1'), stderr=subprocess.DEVNULL)
self.assertEqual(
output1.splitlines(),
[
'Software Release URL: http://sr1//',
'Instance state: busy',
'Instance parameters:',
"{'couscous': 'hello'}",
'Connection parameters:',
'None'
],
)
try:
output2 = self.cliDoSlapos(('service', 'info', 'MyInstance2'), stderr=subprocess.STDOUT)
self.fail()
except subprocess.CalledProcessError as e:
self.assertIn('Instance MyInstance2 does not exist.', e.output)
def test_invalid_service_names(self):
invalid_names = ('"MyInstance0', 'MyInstance1"', 'My"Instance2', 'title:="MyInstance3"', 'reference:="MyInstance4"')
self.format_for_number_of_partitions(len(invalid_names))
for i, name in enumerate(invalid_names):
self.request('http://sr%d//' % i, None, name)
for i, name in enumerate(invalid_names):
try:
self.cliDoSlapos(('service', 'info', name), method=subprocess.check_output, stderr=subprocess.STDOUT)
self.fail()
except subprocess.CalledProcessError as e:
self.assertIn("HTTPError: 400 Client Error: BAD REQUEST", e.output)
def test_invalid_computer_names(self):
invalid_names = ('"COMP-0', 'COMP-1"', 'COMP"-2', 'title:="COMP-3"', 'reference:="COMP-4"')
for name in invalid_names:
self.format_for_number_of_partitions(1, name)
for name in invalid_names:
try:
self.cliDoSlapos(('computer', 'info', name), method=subprocess.check_output, stderr=subprocess.STDOUT)
self.fail()
except subprocess.CalledProcessError as e:
self.assertIn("HTTPError: 400 Client Error: BAD REQUEST", e.output)
class TestMultiNodeSupport(MasterMixin): class TestMultiNodeSupport(MasterMixin):
def test_multi_node_support_different_software_release_list(self): def test_multi_node_support_different_software_release_list(self):
""" """
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment