wip code server

software/code-server/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[instance]
+filename = instance.cfg.in
+md5sum = d41d8cd98f00b204e9800998ecf8427e
+
+[python-language-server-requirements.txt]
+filename = python-language-server-requirements.txt
+md5sum = 0883a40ebcb33d8d7c520490b21bd16c

software/code-server/python-language-server-requirements.txt

+appdirs==1.4.3
+astroid==2.3.3
+attrs==18.2.0
+black==18.9b0
+Click==7.0
+future==0.17.1
+isort==4.3.21
+jedi==0.13.2
+lazy-object-proxy==1.4.3
+mccabe==0.6.1
+mypy==0.770
+mypy-extensions==0.4.3
+parso==0.3.2
+pluggy==0.8.1
+pydocstyle==3.0.0
+pyflakes==2.1.0
+pygls==0.9.1
+pylint==2.4.4
+python-jsonrpc-server==0.1.2
+-e git+https://github.com/palantir/python-language-server@50d03d5931d564e9908292ccfa21dd629ee817ba#egg=python_language_server
+rope==0.11.0
+six==1.12.0
+snowballstemmer==1.2.1
+toml==0.10.0
+typed-ast==1.4.1
+typing-extensions==3.7.4.2
+wrapt==1.11.2
+yapf==0.29.0
+zc.buildout.languageserver==0.4.0
+theia-open==0.3.0

software/code-server/software.cfg

+[buildout]
+extends =
+   ../../component/caddy/buildout.cfg
+   ../../component/jq/buildout.cfg
+   ../../component/git/buildout.cfg
+   ../../component/bash/buildout.cfg
+   ../../component/bash-completion/buildout.cfg
+   ../../component/fish-shell/buildout.cfg
+   ../../component/tmux/buildout.cfg
+   ../../component/tig/buildout.cfg
+   ../../component/vim/buildout.cfg
+   ../../component/curl/buildout.cfg
+   ../../component/coreutils/buildout.cfg
+   ../../component/glib/buildout.cfg
+   ../../component/gnutls/buildout.cfg
+   ../../component/xorg/buildout.cfg
+   ../../component/pkgconfig/buildout.cfg
+   ../../component/fonts/buildout.cfg
+   ../../stack/slapos.cfg
+   ../../stack/monitor/buildout.cfg
+   ../../stack/nodejs.cfg
+   ../../component/defaults.cfg
+   ./buildout.hash.cfg
+
+parts =
+  code-server-wrapper
+
+# default for slapos-standalone
+shared-part-list =
+
+# We keep the gcc part in sync with the one from erp5 software, so that when we install
+# erp5 inside embedded slapos parts can be shared.
+[gcc]
+max_version = 0
+
+[python]
+part = python3
+
+[nodejs]
+<= nodejs-12.18.3
+
+[yarn]
+<= yarn-1.22.10
+
+# sudo apt-get install build-essential g++ libx11-dev libxkbfile-dev libsecret-1-dev python-is-python3
+
+[code-server-repository]
+recipe = slapos.recipe.build:gitclone
+repository = https://github.com/cdr/code-server
+revision = v3.9.0
+# 1.53.2-0-g622cb03f7e
+
+# TODO: libxkbfile shared ?
+[libxkbfile]
+shared = true
+
+[libsecret]
+; XXX old version which does not require meson
+recipe = slapos.recipe.cmmi
+url = https://download.gnome.org/sources/libsecret/0.18/libsecret-0.18.8.tar.xz
+md5sum = fee403988442f497f3c1dfe7b128869c
+shared = true
+configure-options = 
+  --with-libgcrypt-prefix=${libgcrypt:location}
+  --disable-gtk-doc-html
+  --disable-manpages
+environment = 
+  PATH=${xz-utils:location}/bin/:${pkgconfig:location}/bin:${libxslt:location}/bin:${glib:location}/bin:%(PATH)s
+  PKG_CONFIG_PATH=${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig
+
+[code-server]
+recipe = plone.recipe.command
+# TODO:
+# ./lib/coder-cloud-agent
+
+command =
+  bash -c "export TMPDIR=${:location}/tmp \
+        LDFLAGS=-L${gettext:location}/lib \
+        PKG_CONFIG_PATH=${libsecret:location}/lib/pkgconfig:${glib:location}/lib/pkgconfig:${pcre:location}/lib/pkgconfig:${libX11:location}/lib/pkgconfig:${libxkbfile:location}/lib/pkgconfig/:${xproto:location}/lib/pkgconfig/:${kbproto:location}/lib/pkgconfig/:${libxcb:location}/lib/pkgconfig/:${xorg-libpthread-stubs:location}/lib/pkgconfig/:${libXau:location}/lib/pkgconfig/ \
+        PATH=${pkgconfig:location}/bin/:${jq:location}/bin:$PATH &&
+    mkdir -p ${:location} && \
+    mkdir -p \$TMPDIR && \
+    cd ${code-server-repository:location} && \
+    ${yarn:location}/bin/yarn --frozen-lockfile &&
+    ${yarn:location}/bin/yarn build &&
+    ${yarn:location}/bin/yarn build:vscode &&
+    ${yarn:location}/bin/yarn release"
+# TODO: where ?
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+update = ${:command}
+
+[code-server-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode = 0777
+template =
+  inline:
+  #!/bin/bash
+  export PATH=${python-language-server:location}/bin/:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
+  # https://github.com/cdr/code-server/blob/main/docs/FAQ.md#how-do-i-configure-the-marketplace-url
+  export SERVICE_URL=https://open-vsx.org/vscode/gallery
+  export ITEM_URL=https://open-vsx.org/vscode/item
+
+  # TODO: is this a problem ?
+  # XXX https://github.com/cdr/code-server/blob/main/docs/FAQ.md#how-do-i-securely-access-web-services
+
+  # reset PS1 from gowork XXX
+  export PS1='$ '
+  cd ${code-server-repository:location}/release/
+  echo "TODO: depend on ${code-server:location} ..."
+  
+  exec ${nodejs:location}/bin/node . $@
+# TODO:
+#        --home                 Set a custom link for the 'Go Home' button in the Application Menu
+#       --disable-telemetry    Disable telemetry.
+#       --disable-update-check Disable update check. Without this flag, code-server checks every 6 hours against the latest github release and
+#       --cert                 Path to certificate. A self signed certificate is generated if none is provided.
+#       --cert-key             Path to certificate key when using non-generated cert.
+
+[slapos-standalone]
+recipe = zc.recipe.egg
+eggs =
+  slapos.core
+scripts = ${:_buildout_section_name_}
+script-path = ${buildout:bin-directory}/${:scripts}
+# XXX generate a fake entry point for a non existant module, that will not
+# be used because we exit in initialization step
+entry-points =
+  ${:scripts}=not_used:main
+initialization =
+  import argparse
+  import glob
+  import os.path
+  import sys
+  import signal
+  import time
+
+  import slapos.slap.standalone
+
+  parser = argparse.ArgumentParser()
+  parser.add_argument('base_directory')
+  parser.add_argument('ipv4')
+  parser.add_argument('ipv6')
+  parser.add_argument('server_port', type=int)
+  parser.add_argument('computer_id')
+  forwarded_arguments = parser.add_argument_group('forwarded')
+  forwarded_arguments.add_argument('master_url')
+  forwarded_arguments.add_argument('computer')
+  forwarded_arguments.add_argument('partition')
+  # cert and key are optional
+  forwarded_arguments.add_argument('--cert')
+  forwarded_arguments.add_argument('--key')
+  args = parser.parse_args()
+  shared_part_list = [x.strip() for x in '''${buildout:shared-part-list}'''.splitlines() if x.strip()]
+  partition_forward_configuration = (
+      slapos.slap.standalone.PartitionForwardAsPartitionConfiguration(
+          master_url=args.master_url,
+          computer=args.computer,
+          partition=args.partition,
+          cert=args.cert,
+          key=args.key,
+          software_release_list=(
+            'http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg',
+          ),
+      ),
+  )
+  standalone = slapos.slap.standalone.StandaloneSlapOS(
+      args.base_directory,
+      args.ipv4,
+      args.server_port,
+      computer_id=args.computer_id,
+      shared_part_list=shared_part_list,
+      software_root="%s/software" % args.base_directory,
+      instance_root="%s/instance" % args.base_directory,
+      partition_forward_configuration=partition_forward_configuration,
+  )
+  standalone.start()
+  partition_count = 20
+  if len(glob.glob(os.path.join(standalone.instance_directory, '*'))) < partition_count:
+    print("Standalone SlapOS: Formatting {partition_count} partitions".format(
+        partition_count=partition_count))
+    standalone.format(
+        partition_count,
+        args.ipv4,
+        args.ipv6
+    )
+  print("Standalone SlapOS for computer `{}` started".format(args.computer_id))
+  # Run instance at least once, to start the supervisor managing instances.
+  try:
+    standalone.waitForInstance(max_retry=0)
+  except slapos.slap.standalone.SlapOSNodeCommandError as e:
+    print("Error instanciating: {}".format(e))
+
+  quit_requested = []
+  def signal_handler(signum, frame):
+    print("Signal {signum} received".format(signum=signum))
+    quit_requested.append(True)
+  signal.signal(signal.SIGTERM, signal_handler)
+
+  print("Standalone SlapOS ready")
+  while not quit_requested:
+    time.sleep(.1)
+
+  print("Stopping standalone subsystem")
+  standalone.stop()
+  print("Exiting")
+  sys.exit(0)
+
+needs-these-eggs-scripts-in-path =
+  ${supervisor:recipe}
+  ${slapos-command:recipe}
+
+[supervisor]
+recipe = zc.recipe.egg
+eggs =
+  supervisor
+  setuptools
+
+[template-base]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:parts-directory}/${:_buildout_section_name_}
+mode = 0644
+
+[python-language-server]
+version = 0.19.0
+recipe = plone.recipe.command
+command =
+  PATH=${git:location}/bin/:$PATH  bash -c "${python3:executable} -m venv --clear ${:location} && \
+    . ${:location}/bin/activate && \
+    pip install -r ${python-language-server-requirements.txt:output}"
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+
+[python-language-server-requirements.txt]
+<= template-base
+
+[theia]
+recipe = plone.recipe.command
+command = ${bash:location}/bin/bash -c "
+  export TMPDIR=${:location}/tmp PATH=${nodejs:location}/bin:$PATH &&
+  mkdir -p ${:location} && \
+  mkdir -p \$TMPDIR && \
+  cd ${:location} && \
+  cp ${package.json:rendered} . &&
+  cp ${yarn.lock:output} . &&
+  ${yarn:location}/bin/yarn && \
+  ${yarn:location}/bin/yarn theia build"
+location = ${buildout:parts-directory}/${:_buildout_section_name_}
+stop-on-error = true
+uses = ${yarn.lock:recipe}
+
+[cli-utilities]
+PATH = ${nodejs:location}/bin/:${bash:location}/bin/:${fish-shell:location}/bin/:${tig:location}/bin/:${vim:location}/bin/:${tmux:location}/bin/:${git:location}/bin/:${curl:location}/bin:${python2.7:location}/bin/:${buildout:bin-directory}
+
+[theia-wrapper]
+recipe = slapos.recipe.template:jinja2
+rendered = ${buildout:bin-directory}/${:_buildout_section_name_}
+mode = 0777
+template =
+  inline:
+  #!/bin/bash
+  . ${gowork:env.sh}
+  export PATH=${python-language-server:location}/bin/:${java-jdk:location}/bin/:${cli-utilities:PATH}:$HOME/.cargo/bin:$PATH
+  export THEIA_DEFAULT_PLUGINS="local-dir:${theia-plugins:location}"
+  # reset PS1 from gowork
+  export PS1='$ '
+  cd ${theia:location}
+  exec ${yarn:location}/bin/yarn theia start $@
+
+
+[instance]
+<= template-base
+output = ${buildout:directory}/instance.cfg

software/code-server/test/README.md

+Tests for Theia software release

software/code-server/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.theia'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Theia",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'supervisor',
+        'pexpect',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/code-server/test/test.py

+##############################################################################
+#
+# Copyright (c) 2021 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from __future__ import unicode_literals
+
+import os
+import textwrap
+import logging
+import subprocess
+import tempfile
+import time
+import re
+from six.moves.urllib.parse import urlparse, urljoin
+
+import pexpect
+import psutil
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+from slapos.grid.svcbackend import getSupervisorRPC
+from slapos.grid.svcbackend import _getSupervisordSocketPath
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestCodeServer(SlapOSInstanceTestCase):
+  __partition_reference__ = 'CS' # for sockets in included slapos
+  def setUp(self):
+    self.connection_parameters = self.computer_partition.getConnectionParameterDict()
+
+  def test_http_get(self):
+    resp = requests.get(self.connection_parameters['url'], verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)
+
+    # with login/password, this is allowed
+    parsed_url = urlparse(self.connection_parameters['url'])
+    authenticated_url = parsed_url._replace(
+        netloc='{}:{}@[{}]:{}'.format(
+            self.connection_parameters['username'],
+            self.connection_parameters['password'],
+            parsed_url.hostname,
+            parsed_url.port,
+        )).geturl()
+    resp = requests.get(authenticated_url, verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
+    # there's a public folder to serve file
+    with open('{}/srv/frontend-static/public/test_file'.format(
+        self.computer_partition_root_path), 'w') as f:
+      f.write("hello")
+    resp = requests.get(urljoin(authenticated_url, '/public/'), verify=False)
+    self.assertIn('test_file', resp.text)
+    resp = requests.get(
+        urljoin(authenticated_url, '/public/test_file'), verify=False)
+    self.assertEqual('hello', resp.text)
+
+    # there's a (not empty) favicon
+    resp = requests.get(
+        urljoin(authenticated_url, '/favicon.ico'), verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+    self.assertTrue(resp.raw)
+
+    # there is a CSS referencing fonts
+    css_text = requests.get(urljoin(authenticated_url, '/css/slapos.css'), verify=False).text
+    css_urls = re.findall(r'url\([\'"]+([^\)]+)[\'"]+\)', css_text)
+    self.assertTrue(css_urls)
+    # and fonts are served
+    for url in css_urls:
+      resp = requests.get(urljoin(authenticated_url, url), verify=False)
+      self.assertEqual(requests.codes.ok, resp.status_code)
+      self.assertTrue(resp.raw)
+
+  def test_theia_slapos(self):
+    # Make sure we can use the shell and the integrated slapos command
+    process = pexpect.spawnu(
+        '{}/bin/theia-shell'.format(self.computer_partition_root_path),
+        env={'HOME': self.computer_partition_root_path})
+
+    # use a large enough terminal so that slapos proxy show table fit in the screen
+    process.setwinsize(5000, 5000)
+
+    # log process output for debugging
+    logger = logging.getLogger('theia-shell')
+    class DebugLogFile:
+      def write(self, msg):
+        logger.info("output from theia-shell: %s", msg)
+      def flush(self):
+        pass
+    process.logfile = DebugLogFile()
+
+    process.expect_exact('Standalone SlapOS for computer `slaprunner` activated')
+
+    # try to supply and install a software to check that this slapos is usable
+    process.sendline(
+        'slapos supply https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg slaprunner'
+    )
+    process.expect(
+        'Requesting software installation of https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg...'
+    )
+
+    # we pipe through cat to disable pager and prevent warnings like
+    # WARNING: terminal is not fully functional
+    process.sendline('slapos proxy show | cat')
+    process.expect(
+        'https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg'
+    )
+
+    process.sendline('slapos node software')
+    process.expect(
+        'Installing software release https://lab.nexedi.com/nexedi/slapos/raw/1.0.144/software/helloworld/software.cfg'
+    )
+    # interrupt this, we don't want to actually wait for software installation
+    process.sendcontrol('c')
+
+    process.terminate()
+    process.wait()
+
+  def test_theia_shell_execute_tasks(self):
+    # shell needs to understand -c "command" arguments for theia tasks feature
+    test_file = '{}/test file'.format(self.computer_partition_root_path)
+    subprocess.check_call([
+        '{}/bin/theia-shell'.format(self.computer_partition_root_path),
+        '-c',
+        'touch "{}"'.format(test_file)
+    ])
+    self.assertTrue(os.path.exists(test_file))
+
+
+class TestTheiaEmbeddedSlapOSShutdown(SlapOSInstanceTestCase):
+  __partition_reference__ = 'T' # for sockets in included slapos
+
+  def test_stopping_instance_stops_embedded_slapos(self):
+    embedded_slapos_supervisord_socket = _getSupervisordSocketPath(
+        os.path.join(
+            self.computer_partition_root_path,
+            'srv',
+            'runner',
+            'instance',
+        ), self.logger)
+
+    # Wait a bit for this supervisor to be started.
+    for _ in range(20):
+      if os.path.exists(embedded_slapos_supervisord_socket):
+        break
+      time.sleep(1)
+
+    # get the pid of the supervisor used to manage instances
+    with getSupervisorRPC(embedded_slapos_supervisord_socket) as embedded_slapos_supervisor:
+      embedded_slapos_process = psutil.Process(embedded_slapos_supervisor.getPID())
+
+    # Stop theia's services
+    with self.slap.instance_supervisor_rpc as instance_supervisor:
+      process_info, = [
+          p for p in instance_supervisor.getAllProcessInfo()
+          if p['name'].startswith('slapos-standalone-instance-')
+      ]
+      instance_supervisor.stopProcessGroup(process_info['group'])
+
+    # the supervisor controlling instances is also stopped
+    self.assertFalse(embedded_slapos_process.is_running())