Commit cdd6690d authored by Jérome Perrin's avatar Jérome Perrin

theia: include a slapos

Include a slapos command implemented with slapos standalone

See merge request !743
parents 7bf2cf5a 358b08c5
...@@ -15,7 +15,7 @@ ...@@ -15,7 +15,7 @@
[instance-profile] [instance-profile]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 0edf9e67bc637d73a7415124c8082a8c md5sum = 13f4f3806522b265dd1912ff169d146d
[influxdb-config-file] [influxdb-config-file]
filename = influxdb-config-file.cfg.in filename = influxdb-config-file.cfg.in
......
...@@ -267,6 +267,10 @@ config-url = ${grafana:url} ...@@ -267,6 +267,10 @@ config-url = ${grafana:url}
config-https-only = true config-https-only = true
return = domain secure_access return = domain secure_access
[apache-frontend-available-promise]
<= check-url-available-promise
url = ${apache-frontend:connection-secure_access}
[promises] [promises]
recipe = recipe =
...@@ -277,6 +281,8 @@ instance-promises = ...@@ -277,6 +281,8 @@ instance-promises =
${grafana-listen-promise:path} ${grafana-listen-promise:path}
${loki-listen-promise:path} ${loki-listen-promise:path}
${promtail-listen-promise:path} ${promtail-listen-promise:path}
${promtail-listen-promise:path}
${apache-frontend-available-promise:path}
[publish-connection-parameter] [publish-connection-parameter]
......
...@@ -292,3 +292,5 @@ testfixtures = 6.11 ...@@ -292,3 +292,5 @@ testfixtures = 6.11
funcsigs = 1.0.2 funcsigs = 1.0.2
PyPDF2 = 1.26.0 PyPDF2 = 1.26.0
mysqlclient = 1.3.12 mysqlclient = 1.3.12
pexpect = 4.8.0
ptyprocess = 0.6.0
...@@ -15,12 +15,12 @@ ...@@ -15,12 +15,12 @@
[instance] [instance]
filename = instance.cfg.in filename = instance.cfg.in
md5sum = 21735765808aac82fb91d53341a3c0d6 md5sum = 99dfaa031e58518326b731e9218aa425
[yarn.lock] [yarn.lock]
filename = yarn.lock filename = yarn.lock
md5sum = c2523a5c832f617c374ee621d50d9e52 md5sum = ae1b596804715acd3512f1e8e6cbae3b
[python-language-server-requirements.txt] [python-language-server-requirements.txt]
filename = python-language-server-requirements.txt filename = python-language-server-requirements.txt
md5sum = 6db2a484cac19787fecd87fffefa4aa9 md5sum = 2a296cba4c36d7a5fca5f4347a8c8469
...@@ -50,8 +50,10 @@ template = inline: ...@@ -50,8 +50,10 @@ template = inline:
tls { tls {
alpn http/1.1 alpn http/1.1
} }
root $${directory:frontend-static}
browse
proxy / $${theia-instance:base-url} { proxy / $${theia-instance:base-url} {
# transparent except public $${favicon.ico:filename}
} }
proxy /services $${theia-instance:base-url} { proxy /services $${theia-instance:base-url} {
websocket websocket
...@@ -89,6 +91,29 @@ hash-files = ...@@ -89,6 +91,29 @@ hash-files =
$${frontend-instance:wrapper-path} $${frontend-instance:wrapper-path}
wait-for-files = $${frontend-instance:pidfile} wait-for-files = $${frontend-instance:pidfile}
[favicon.ico]
# generate a pseudo random favicon, different for each instance name.
recipe = slapos.recipe.build
install =
import hashlib, shutil
buildout_offline = self.buildout['buildout']['offline']
self.buildout['buildout']['offline'] = 'false'
try:
gravatar_url = "https://www.gravatar.com/avatar/" + hashlib.md5(
'''$${slap-configuration:root-instance-title}'''
).hexdigest() + "?s=256&d=retro"
shutil.copy(self.download(gravatar_url), '''$${:location}''')
except Exception:
# Because installation should work offline, if we can't download a favicon,
# just ignore this step.
self.logger.exception("Error while downloading favicon, using empty one")
open('''$${:location}''', 'w').close()
finally:
self.buildout['buildout']['offline'] = buildout_offline
location = $${directory:frontend-static}/$${:filename}
filename = $${:_buildout_section_name_}
[user] [user]
recipe = slapos.cookbook:userinfo recipe = slapos.cookbook:userinfo
...@@ -96,7 +121,7 @@ recipe = slapos.cookbook:userinfo ...@@ -96,7 +121,7 @@ recipe = slapos.cookbook:userinfo
recipe = slapos.cookbook:wrapper recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:services}/$${:_buildout_section_name_} wrapper-path = $${directory:services}/$${:_buildout_section_name_}
command-line = command-line =
env LC_ALL=C.UTF-8 TMP=$${directory:tmp} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port} env LC_ALL=C.UTF-8 TMP=$${directory:tmp} THEIA_SHELL=$${theia-shell:wrapper-path} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port} $${directory:project}
ip = $${instance-parameter:ipv4-random} ip = $${instance-parameter:ipv4-random}
hostname = $${:ip} hostname = $${:ip}
...@@ -106,11 +131,37 @@ hash-existing-files = ...@@ -106,11 +131,37 @@ hash-existing-files =
${yarn.lock:output} ${yarn.lock:output}
${theia-wrapper:rendered} ${theia-wrapper:rendered}
[theia-shell]
recipe = slapos.cookbook:wrapper
wrapper-path = $${directory:bin}/$${:_buildout_section_name_}
# reset GIT_EXEC_PATH to workaround https://github.com/eclipse-theia/theia/issues/7555
# activate slapos configuration
command-line =
${bash:location}/bin/bash -c ". $${slapos-standalone-activate:rendered} && exec env GIT_EXEC_PATH= ${bash:location}/bin/bash"
[slapos-standalone-activate]
recipe = slapos.recipe.template:jinja2
rendered = $${directory:bin}/$${:_buildout_section_name_}
mode = 0700
# XXX maybe standalone slapos should provide an activate script like virtualenv is doing?
template =
inline:#!/bin/sh
export PATH=${buildout:bin-directory}:$PATH
${slapos-standalone:script-path} $${directory:slapos} $${:ipv4} $${:ipv6} $${:port}
export SLAPOS_CONFIGURATION=$${directory:slapos}/etc/slapos.cfg
export SLAPOS_CLIENT_CONFIGURATION=$SLAPOS_CONFIGURATION
ipv4 = $${instance-parameter:ipv4-random}
ipv6 = $${instance-parameter:ipv6-random}
port = 4000
[promises] [promises]
recipe = recipe =
instance-promises = instance-promises =
$${theia-listen-promise:name} $${theia-listen-promise:name}
$${frontend-listen-promise:name} $${frontend-listen-promise:name}
$${apache-frontend-url-available-promise:name}
[theia-listen-promise] [theia-listen-promise]
<= monitor-promise-base <= monitor-promise-base
...@@ -126,6 +177,13 @@ name = $${:_buildout_section_name_}.py ...@@ -126,6 +177,13 @@ name = $${:_buildout_section_name_}.py
config-hostname = $${frontend-instance:ip} config-hostname = $${frontend-instance:ip}
config-port = $${frontend-instance:port} config-port = $${frontend-instance:port}
[apache-frontend-url-available-promise]
<= monitor-promise-base
module = check_url_available
name = $${:_buildout_section_name_}.py
config-url = $${apache-frontend:connection-secure_access}
config-check-secure = 1
[apache-frontend] [apache-frontend]
<= slap-connection <= slap-connection
recipe = slapos.cookbook:requestoptional recipe = slapos.cookbook:requestoptional
...@@ -163,6 +221,7 @@ tmp = $${buildout:directory}/tmp ...@@ -163,6 +221,7 @@ tmp = $${buildout:directory}/tmp
pidfiles = $${:var}/run pidfiles = $${:var}/run
services = $${:etc}/service services = $${:etc}/service
framebuffer = $${:srv}/framebuffer project = $${:srv}/project
fonts = $${:srv}/fonts slapos = $${:srv}/slapos
home = $${:srv}/home frontend-static = $${:srv}/frontend-static
frontend-static-public = $${:frontend-static}/public
...@@ -26,4 +26,4 @@ typed-ast==1.4.1 ...@@ -26,4 +26,4 @@ typed-ast==1.4.1
typing-extensions==3.7.4.2 typing-extensions==3.7.4.2
wrapt==1.11.2 wrapt==1.11.2
yapf==0.29.0 yapf==0.29.0
zc.buildout.languageserver==0.2.0 zc.buildout.languageserver==0.2.1
...@@ -4,6 +4,11 @@ extends = ...@@ -4,6 +4,11 @@ extends =
../../component/caddy/buildout.cfg ../../component/caddy/buildout.cfg
../../component/git/buildout.cfg ../../component/git/buildout.cfg
../../component/bash/buildout.cfg ../../component/bash/buildout.cfg
../../component/fish-shell/buildout.cfg
../../component/tmux/buildout.cfg
../../component/tig/buildout.cfg
../../component/vim/buildout.cfg
../../component/curl/buildout.cfg
../../component/coreutils/buildout.cfg ../../component/coreutils/buildout.cfg
../../stack/slapos.cfg ../../stack/slapos.cfg
../../stack/monitor/buildout.cfg ../../stack/monitor/buildout.cfg
...@@ -20,6 +25,9 @@ parts = ...@@ -20,6 +25,9 @@ parts =
slapos-cookbook slapos-cookbook
instance instance
# default for slapos-standalone
shared-part-list =
[nodejs] [nodejs]
<= nodejs-10.19.0 <= nodejs-10.19.0
...@@ -33,12 +41,66 @@ recipe = slapos.recipe.build:download-unpacked ...@@ -33,12 +41,66 @@ recipe = slapos.recipe.build:download-unpacked
url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
md5sum = 4a02e1687a150113ad6b0215f9afdb3e md5sum = 4a02e1687a150113ad6b0215f9afdb3e
[slapos-standalone]
recipe = zc.recipe.egg
eggs =
slapos.core
scripts = ${:_buildout_section_name_}
script-path = ${buildout:bin-directory}/${:scripts}
# XXX generate a fake entry point for a non existant module, that will not
# be used because we exit in initialization step
entry-points =
${:scripts}=not_used:main
initialization =
import argparse
import os.path
import sys
import glob
import slapos.slap.standalone
parser = argparse.ArgumentParser()
parser.add_argument('base_directory')
parser.add_argument('ipv4')
parser.add_argument('ipv6')
parser.add_argument('server_port', type=int)
args = parser.parse_args()
shared_part_list = [x.strip() for x in '''${buildout:shared-part-list}'''.splitlines() if x.strip()]
standalone = slapos.slap.standalone.StandaloneSlapOS(
args.base_directory,
args.ipv4,
args.server_port,
shared_part_list=shared_part_list
)
standalone.start()
partition_count = 20
if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
print("Standalone SlapOS: Formatting {partition_count} partitions".format(
partition_count=partition_count))
standalone.format(
partition_count,
args.ipv4,
args.ipv6
)
print ("Standalone SlapOS for computer `{}` activated".format(standalone.computer._computer_id))
sys.exit(0)
needs-these-eggs-scripts-in-path =
${supervisor:recipe}
${slapos-command:recipe}
[supervisor]
recipe = zc.recipe.egg
eggs =
supervisor
setuptools
[python-language-server] [python-language-server]
version = 0.19.0 version = 0.19.0
recipe = plone.recipe.command recipe = plone.recipe.command
command = command =
bash -c "${python3:executable} -m venv ${:location} && \ PATH=${git:location}/bin/:$PATH bash -c "${python3:executable} -m venv ${:location} && \
. ${:location}/bin/activate && \ . ${:location}/bin/activate && \
pip install -r ${python-language-server-requirements.txt:output}" pip install -r ${python-language-server-requirements.txt:output}"
location = ${buildout:parts-directory}/${:_buildout_section_name_} location = ${buildout:parts-directory}/${:_buildout_section_name_}
...@@ -77,9 +139,7 @@ mode = 0644 ...@@ -77,9 +139,7 @@ mode = 0644
[package.json] [package.json]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
# this comes from https://github.com/theia-ide/theia-apps/blob/0cc6a3dc2a6dec1b2b4f9572e9d878a67dc81ff5/theia-full-docker/latest.package.json # this comes from https://github.com/theia-ide/theia-apps/blob/2991e3a433f031b22bc80e274f80620d1e6898e5/theia-full-docker/latest.package.json
# but we pin vscode-json-languageserver to 1.2.2, otherwise 1.2.3 is picked and it seems to be incompatible.*
# See https://github.com/theia-ide/theia-apps/issues/333
template = template =
inline:{ inline:{
"private": true, "private": true,
...@@ -145,6 +205,7 @@ template = ...@@ -145,6 +205,7 @@ template =
"@theia/terminal": "latest", "@theia/terminal": "latest",
"@theia/userstorage": "latest", "@theia/userstorage": "latest",
"@theia/variable-resolver": "latest", "@theia/variable-resolver": "latest",
"@theia/vsx-registry": "latest",
"@theia/workspace": "latest" "@theia/workspace": "latest"
}, },
"resolutions": { "resolutions": {
...@@ -226,7 +287,7 @@ template = ...@@ -226,7 +287,7 @@ template =
"vscode-java-dependency-viewer": "https://github.com/microsoft/vscode-java-dependency/releases/download/0.6.0/vscode-java-dependency-0.6.0.vsix", "vscode-java-dependency-viewer": "https://github.com/microsoft/vscode-java-dependency/releases/download/0.6.0/vscode-java-dependency-0.6.0.vsix",
"vscode-java-redhat": "https://github.com/redhat-developer/vscode-java/releases/download/v0.54.2/redhat.java-0.54.2.vsix", "vscode-java-redhat": "https://github.com/redhat-developer/vscode-java/releases/download/v0.54.2/redhat.java-0.54.2.vsix",
"vscode-java-test": "https://github.com/microsoft/vscode-java-test/releases/download/0.22.0/vscjava.vscode-java-test-0.22.0.vsix", "vscode-java-test": "https://github.com/microsoft/vscode-java-test/releases/download/0.22.0/vscjava.vscode-java-test-0.22.0.vsix",
"vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.1.58038/ms-python-release.vsix", "vscode-python": "https://github.com/microsoft/vscode-python/releases/download/2020.4.74986/ms-python-release.vsix",
"vscode-ruby": "https://github.com/rubyide/vscode-ruby/releases/download/v0.25.0/ruby-0.25.0.vsix", "vscode-ruby": "https://github.com/rubyide/vscode-ruby/releases/download/v0.25.0/ruby-0.25.0.vsix",
"vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.2.0/vscode-zc-buildout-0.2.0.vsix" "vscode-zc-buildout": "https://github.com/perrinjerome/vscode-zc-buildout/releases/download/v0.2.0/vscode-zc-buildout-0.2.0.vsix"
} }
...@@ -257,6 +318,8 @@ install += ...@@ -257,6 +318,8 @@ install +=
github.com/haya14busa/goplay/cmd/goplay github.com/haya14busa/goplay/cmd/goplay
github.com/davidrjenni/reftools/cmd/fillstruct github.com/davidrjenni/reftools/cmd/fillstruct
[cli-utilities]
PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin
[theia-wrapper] [theia-wrapper]
recipe = slapos.recipe.template:jinja2 recipe = slapos.recipe.template:jinja2
...@@ -265,10 +328,9 @@ mode = 0777 ...@@ -265,10 +328,9 @@ mode = 0777
template = template =
inline: inline:
#!/bin/bash #!/bin/bash
export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
. ${gowork:env.sh} . ${gowork:env.sh}
export PATH=${python-language-server:location}/bin/:${cli-utilities:PATH}:$PATH
export THEIA_DEFAULT_PLUGINS="local-dir:${theia:THEIA_DEFAULT_PLUGINS}" export THEIA_DEFAULT_PLUGINS="local-dir:${theia:THEIA_DEFAULT_PLUGINS}"
export SHELL=bash
# reset PS1 from gowork # reset PS1 from gowork
export PS1='$ ' export PS1='$ '
cd ${theia:location} cd ${theia:location}
......
...@@ -45,6 +45,7 @@ setup( ...@@ -45,6 +45,7 @@ setup(
'slapos.libnetworkcache', 'slapos.libnetworkcache',
'erp5.util', 'erp5.util',
'supervisor', 'supervisor',
'pexpect',
'requests', 'requests',
], ],
zip_safe=True, zip_safe=True,
......
...@@ -24,25 +24,27 @@ ...@@ -24,25 +24,27 @@
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
# #
############################################################################## ##############################################################################
from __future__ import unicode_literals
import os import os
import textwrap import textwrap
import logging import logging
import tempfile import tempfile
import time import time
from six.moves.urllib.parse import urlparse from six.moves.urllib.parse import urlparse, urljoin
import pexpect
import requests import requests
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass( setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
os.path.abspath( os.path.abspath(
os.path.join(os.path.dirname(__file__), '..', 'software.cfg'))) os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
class TestTheia(SlapOSInstanceTestCase): class TestTheia(SlapOSInstanceTestCase):
__partition_reference__ = 'T' # for sockets in included slapos
def setUp(self): def setUp(self):
self.connection_parameters = self.computer_partition.getConnectionParameterDict() self.connection_parameters = self.computer_partition.getConnectionParameterDict()
...@@ -52,12 +54,71 @@ class TestTheia(SlapOSInstanceTestCase): ...@@ -52,12 +54,71 @@ class TestTheia(SlapOSInstanceTestCase):
# with login/password, this is allowed # with login/password, this is allowed
parsed_url = urlparse(self.connection_parameters['url']) parsed_url = urlparse(self.connection_parameters['url'])
resp = requests.get( authenticated_url = parsed_url._replace(
parsed_url._replace(
netloc='{}:{}@[{}]:{}'.format( netloc='{}:{}@[{}]:{}'.format(
self.connection_parameters['username'], self.connection_parameters['username'],
self.connection_parameters['password'], self.connection_parameters['password'],
parsed_url.hostname, parsed_url.hostname,
parsed_url.port)).geturl(), parsed_url.port,
verify=False) )).geturl()
resp = requests.get(authenticated_url, verify=False)
self.assertEqual(requests.codes.ok, resp.status_code)
# there's a public folder to serve file
with open('{}/srv/frontend-static/public/test_file'.format(
self.computer_partition_root_path), 'w') as f:
f.write("hello")
resp = requests.get(urljoin(authenticated_url, '/public/'), verify=False)
self.assertIn('test_file', resp.text)
resp = requests.get(
urljoin(authenticated_url, '/public/test_file'), verify=False)
self.assertEqual('hello', resp.text)
# there's a (not empty) favicon
resp = requests.get(
urljoin(authenticated_url, '/favicon.ico'), verify=False)
self.assertEqual(requests.codes.ok, resp.status_code) self.assertEqual(requests.codes.ok, resp.status_code)
self.assertTrue(resp.raw)
def test_theia_slapos(self):
# Make sure we can use the shell and the integrated slapos command
process = pexpect.spawnu(
'{}/bin/theia-shell'.format(self.computer_partition_root_path),
env={'HOME': self.computer_partition_root_path})
# use a large enough terminal so that slapos proxy show table fit in the screen
process.setwinsize(5000, 5000)
process.expect_exact('Standalone SlapOS: Formatting 20 partitions')
process.expect_exact('Standalone SlapOS for computer `local` activated')
# try to supply and install a software to check that this slapos is usable
process.sendline(
'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg local'
)
process.expect(
'Requesting software installation of https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg...'
)
# we pipe through cat to disable pager and prevent warnings like
# WARNING: terminal is not fully functional
process.sendline('slapos proxy show | cat')
process.expect(
'https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg'
)
process.sendline('slapos node software')
process.expect(
'Installing software release https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg'
)
# interrupt this, we don't want to actually wait for software installation
process.sendcontrol('c')
# shutdown this slapos
process.sendline(
'supervisorctl -c {}/srv/slapos/etc/supervisord.conf shutdown'.format(
self.computer_partition_root_path))
process.expect('Shut down')
process.terminate()
process.wait()
This source diff could not be displayed because it is too large. You can view the blob instead.
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment