Update theia

vscode's Python extension works on this version.

/reviewed-on nexedi/slapos!643

component/nodejs/buildout.cfg

@@ -15,9 +15,14 @@ parts =
 [nodejs]
 <= nodejs-8.9.4
 
+[nodejs-10.6.0]
+<= nodejs-base
+openssl_location = ${openssl:location}
+version = v10.6.0
+md5sum = 9df233b86244ebda1ded1f91694fbe86
+
 [nodejs-8.9.4]
 <= nodejs-base
-recipe = slapos.recipe.cmmi
 version = v8.9.4
 md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
 
@@ -26,24 +31,27 @@ md5sum = 4ddc1daff327d7e6f63da57fdfc24f55
 version = v8.6.0
 md5sum = 0c95e08220667d8a18b97ecec8218ac6
 
+
 [nodejs-base]
 # Server-side Javascript.
 version =
 md5sum =
+openssl_location = ${openssl-1.0:location}
 
 recipe = slapos.recipe.cmmi
+shared = true
 url = https://nodejs.org/dist/${:version}/node-${:version}.tar.gz
 configure-options =
   --shared-openssl
-  --shared-openssl-includes=${openssl-1.0:location}/include
-  --shared-openssl-libpath=${openssl-1.0:location}/lib
+  --shared-openssl-includes=${:openssl_location}/include
+  --shared-openssl-libpath=${:openssl_location}/lib
 environment =
-  HOME=${buildout:parts-directory}/${:_buildout_section_name_}
+  HOME=@@LOCATION@@
   PATH=${gcc:location}/bin:${pkgconfig:location}/bin:${python2.7:location}/bin/:%(PATH)s
-  PKG_CONFIG_PATH=${openssl-1.0:location}/lib/pkgconfig/
+  PKG_CONFIG_PATH=${:openssl_location}/lib/pkgconfig/
   CPPFLAGS=-I${zlib:location}/include
-  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -Wl,-rpath=${openssl-1.0:location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
-  LD_LIBRARY_PATH=${openssl-1.0:location}/lib
+  LDFLAGS=-Wl,-rpath=${gcc:location}/lib -Wl,-rpath=${gcc:location}/lib64 -Wl,-rpath=${:openssl_location}/lib -L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LD_LIBRARY_PATH=${:openssl_location}/lib
 
 [nodejs-8.6.0-output]
 # Shared binary location to ease migration

software/slapos-sr-testing/buildout.hash.cfg

@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 3d12778d8dc910180ff3a7fcae8b37f9
+md5sum = 4664f7dae66d3f582e34cec2ca627501

software/slapos-sr-testing/instance.cfg

@@ -28,7 +28,7 @@ bin = $${buildout:directory}/bin
 working-dir = $${buildout:directory}/tmp
 
 [test-list]
-path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup}
+path_list = ${slapos.cookbook-setup:setup},${slapos.test.caddy-frontend-setup:setup},${slapos.test.erp5-setup:setup},${slapos.test.slapos-master-setup:setup},${slapos.test.kvm-setup:setup},${slapos.test.monitor-setup:setup},${slapos.test.plantuml-setup:setup},${slapos.test.powerdns-setup:setup},${slapos.test.proftpd-setup:setup},${slapos.test.re6stnet-setup:setup},${slapos.test.seleniumserver-setup:setup},${slapos.test.slaprunner-setup:setup},${slapos.test.helloworld-setup:setup},${slapos.test.jupyter-setup:setup},${slapos.test.nextcloud-setup:setup},${slapos.test.turnserver-setup:setup},${slapos.test.theia-setup:setup}
 
 [slapos-test-runner]
 recipe = slapos.cookbook:wrapper

software/slapos-sr-testing/software.cfg

@@ -102,6 +102,11 @@ setup = ${slapos-repository:location}/software/nextcloud/test/
 egg = slapos.test.nextcloud
 setup = ${slapos-repository:location}/software/turnserver/test/
 
+[slapos.test.theia-setup]
+<= setup-develop-egg
+egg = slapos.test.theia
+setup = ${slapos-repository:location}/software/theia/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git

software/theia/README.md

+# Theia software release
+
+Theia is a cloud (and desktop) IDE https://www.theia-ide.org
+
+This version comes pre-configured with a few plugins, but does not come with python plugin, to let
+you choose between theia and vscode one.
+
+## Installing vscode extensions
+
+To install vscode's python plugin ( https://code.visualstudio.com/docs/languages/python ):
+
+<key>F1</key> `Deploy plugin by id` and enter `vscode:extension/ms-python.python` and wait for a while.
+Check theia's log files in case of issues.
+
+see https://github.com/theia-ide/theia/wiki/Testing-VS-Code-extensions for more details.
+
+## jedi
+
+[jedi](https://github.com/davidhalter/jedi) which is used by both thiea and vscode python plugins has
+some support for `zc.buildout`. It looks up for a `buildout.cfg` file and if found will load all scripts
+from the bin directory from this buildout to add eggs to sys.path. In webrunner we have almost 100 scripts
+in bin directory, with maybe 30 eggs in each scripts, so this makes jedi so slow it's unusable. Also, if
+an error occurs parsing these scripts, jedi won't be usable. This issue is tracked in
+https://github.com/davidhalter/jedi/issues/1325
+
+A simple workaround is to create and empty `buildout.cfg` file at the root of project folder.
\ No newline at end of file

software/theia/buildout.hash.cfg

@@ -15,12 +15,12 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = 6a0a65ff9f81f907d53c42c65b54442a
+md5sum = 7c9444fbe8dc8faea67ede2b77e188ed
 
 [yarn.lock]
 filename = yarn.lock
-md5sum = fc2f20d496911241a0d9001c445d6158
+md5sum = b63a993a13e7c3b16b66c25fac5ac8b9
 
 [python-language-server-requirements.txt]
 filename = python-language-server-requirements.txt
-md5sum = 242c8359be6893725ad2ac8e3c864eb0
+md5sum = ce87c9818d64a16fce49c84ed5dba8fc

software/theia/instance.cfg.in

@@ -2,7 +2,6 @@
 parts =
   monitor-base
   promises
-  .bashrc
   frontend-reload
   publish-connection-parameter
 
@@ -79,37 +78,15 @@ recipe = slapos.cookbook:userinfo
 recipe = slapos.cookbook:wrapper
 wrapper-path = $${directory:services}/$${:_buildout_section_name_}
 command-line =
-  env -i HOME=$${directory:home} LC_ALL=C.UTF-8 USER=$${user:pw-name} LOGNAME=$${user:pw-name} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
+  env LC_ALL=C.UTF-8 TMP=$${directory:tmp} ${theia-wrapper:rendered} --hostname=$${:hostname} --port=$${:port}
 
 ip =  $${instance-parameter:ipv4-random}
 hostname =  $${:ip}
 port = 3000
 base-url = http://$${:hostname}:$${:port}/
-
-[.bashrc]
-recipe = slapos.recipe.template:jinja2
-rendered = $${directory:home}/$${:_buildout_section_name_}
-template = inline:
-  export PS1="$ " # because we are in a gowork workspace
-  # XXX .bash_profile is not executed, so we introduce a bashrc.theia file
-  # to allow customizations.
-  if [ -f "$HOME/.bashrc.theia" ] ; then
-    source $HOME/.bashrc.theia
-  fi
-  # If not running interactively, don't do anything
-  case $- in
-      *i*) ;;
-        *) return;;
-  esac
-  # enable color support
-  if [ -x /usr/bin/dircolors ]; then
-      test -r ~/.dircolors && eval "$(dircolors -b ~/.dircolors)" || eval "$(dircolors -b)"
-      alias ls='ls --color=auto'
-  fi
-  # common ls aliases
-  alias ll='ls -l'
-  alias la='ls -A'
-  alias l='ls -CF'
+hash-existing-files =
+  ${yarn.lock:output}
+  ${theia-wrapper:rendered}
 
 [promises]
 recipe =

software/theia/python-language-server-requirements.txt

@@ -20,4 +20,4 @@ snowballstemmer==1.2.1
 toml==0.10.0
 typed-ast==1.2.0
 typeshed==0.0.1
-yapf==0.25.0
+yapf==0.28.0

software/theia/software.cfg

@@ -24,7 +24,7 @@ parts =
 
 
 [nodejs]
-<= nodejs-8.9.4
+<= nodejs-10.6.0
 
 [python3]
 <= python3.6.6
@@ -34,10 +34,10 @@ parts =
 # this could become a component, but it needs to be invoked from nodejs explicitly,
 # otherwise it uses system's nodejs
 # XXX why don't we build a wrapper ?
-version = 1.11.0
+version = 1.17.3
 recipe = slapos.recipe.build:download-unpacked
 url = https://github.com/yarnpkg/yarn/releases/download/v${:version}/yarn-v${:version}.tar.gz
-md5sum = d4f05075f534dd9a0a8c18c650b55f0d
+md5sum = 4a02e1687a150113ad6b0215f9afdb3e
 
 
 [python-language-server]
@@ -69,6 +69,7 @@ command = ${bash:location}/bin/bash -c "
   ${yarn:location}/bin/yarn theia build"
 location = ${buildout:parts-directory}/${:_buildout_section_name_}
 stop-on-error = true
+uses = ${yarn.lock:recipe}
 
 [yarn.lock]
 recipe = slapos.recipe.template
@@ -78,25 +79,54 @@ mode = 0644
 
 [package.json]
 recipe = slapos.recipe.template:jinja2
-# this comes from https://github.com/theia-ide/theia-apps/blob/598d4dc9d4f9e0514869273c069f734a41f20207/theia-full-docker/next.package.json
+# this comes from https://github.com/theia-ide/theia-apps/blob/def6d29a5fb3739aebc17c6a5b67f1b5c16caacd/theia-full-docker/next.package.json
 # but "@theia/php": "next" was removed, because we don't have php/composer component so installation failed
 template =
   inline:
     {
       "private": true,
+      "theia": {
+          "frontend": {
+              "config": {
+                  "applicationName": "Theia SlapOS",
+                  "preferences": {
+                      "files.enableTrash": false,
+                      "files.exclude": {
+                          "**.pyc": true,
+                          "**.egg-info": true,
+                          "__pycache__": true,
+                          ".git": true,
+                          ".env": true
+                      },
+                      "files.watcherExclude": {
+                        "**/.eggs/**": true,
+                        "**/.env/**": true,
+                        "**/.git/**": true,
+                        "**/node_modules/**": true
+                      },
+                      "editor.multiCursorModifier": "ctrlCmd",
+                      "plantuml.webservice": "//plantuml.host.vifib.net/svg/",
+                      "plantuml.monochrome": false,
+                      "editor.tabSize": 2
+                  }
+              }
+          }
+      },
       "dependencies": {
-      "typescript": "latest",
           "@theia/callhierarchy": "next",
           "@theia/core": "next",
           "@theia/cpp": "next",
+          "@theia/debug": "next",
+          "@theia/debug-nodejs": "next",
           "@theia/docker": "next",
           "@theia/editor": "next",
           "@theia/editorconfig": "next",
-      "@theia/extension-manager": "next",
+          "@theia/editor-preview": "next",
           "@theia/file-search": "next",
           "@theia/filesystem": "next",
           "@theia/git": "next",
           "@theia/go": "next",
+          "@theia/getting-started": "next",
           "@theia/java": "next",
           "@theia/json": "next",
           "@theia/keymaps": "next",
@@ -111,10 +141,12 @@ template =
           "@theia/outline-view": "next",
           "@theia/output": "next",
           "@theia/plantuml": "next",
+          "@theia/plugin": "next",
+          "@theia/plugin-ext": "next",
+          "@theia/plugin-ext-vscode": "next",
           "@theia/preferences": "next",
           "@theia/preview": "next",
           "@theia/process": "next",
-      "@theia/python": "next",
           "@theia/ruby": "next",
           "@theia/rust": "next",
           "@theia/search-in-workspace": "next",
@@ -126,7 +158,8 @@ template =
           "@theia/userstorage": "next",
           "@theia/variable-resolver": "next",
           "@theia/workspace": "next",
-      "theia-yang-extension": "next"
+          "theia-yang-extension": "next",
+          "typescript": "latest"
       },
       "devDependencies": {
           "@theia/cli": "next"
@@ -169,6 +202,8 @@ template =
   export PATH=${nodejs:location}/bin/:${python-language-server:location}/bin/:${bash:location}/bin/:${git:location}/bin/:$PATH
   . ${gowork:env.sh}
   export SHELL=bash
+  # reset PS1 from gowork
+  export PS1='$ '
   cd ${theia:location}
   exec ${yarn:location}/bin/yarn theia start $@
 

software/theia/test/README.md

+Tests for Theia software release

software/theia/test/setup.py

+##############################################################################
+#
+# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+from setuptools import setup, find_packages
+
+version = '0.0.1.dev0'
+name = 'slapos.test.theia'
+long_description = open("README.md").read()
+
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' Theia",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'supervisor',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/theia/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import textwrap
+import logging
+import tempfile
+import time
+from six.moves.urllib.parse import urlparse
+
+import requests
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestTheia(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.theia_url = self.computer_partition.getConnectionParameterDict(
+    )['url']
+
+  def test_http_get(self):
+    resp = requests.get(self.theia_url, verify=False)
+    self.assertEqual(requests.codes.ok, resp.status_code)
+
+    # without login/password, this is unauthorized
+    parsed_url = urlparse(self.theia_url)
+    resp = requests.get(
+        parsed_url._replace(
+            netloc='[{}]:{}'.format(
+                parsed_url.hostname,
+                parsed_url.port)).geturl(),
+        verify=False)
+    self.assertEqual(requests.codes.unauthorized, resp.status_code)