Update Release Candidate

component/ffmpeg/buildout.cfg

@@ -62,10 +62,12 @@ url = http://www.tortall.net/projects/yasm/releases/yasm-1.1.0.tar.gz
 recipe = slapos.recipe.cmmi
 url = https://storage.googleapis.com/downloads.webmproject.org/releases/webm/libvpx-1.5.0.tar.bz2
 md5sum = 49e59dd184caa255886683facea56fca
+location = @@LOCATION@@
 configure-options =
   --enable-shared
 environment =
   PATH=${yasm:location}/bin:%(PATH)s
+  LDFLAGS=-Wl,-rpath=${:location}/lib
 
 [libx264]
 recipe = slapos.recipe.cmmi
@@ -96,6 +98,7 @@ configure-options =
 recipe = slapos.recipe.cmmi
 url = https://ffmpeg.org/releases/ffmpeg-4.1.4.tar.bz2
 md5sum = 611d171e4aee749b85e04d17e2aee71d
+location = @@LOCATION@@
 pkg_config_depends = ${libxcb:location}/lib/pkgconfig:${libxcb:pkg_config_depends}:${libtheora:location}/lib/pkgconfig:${libtheora:pkg_config_depends}:${libvpx:location}/lib/pkgconfig:${libx264:location}/lib/pkgconfig:${opencore-amr:location}/lib/pkgconfig:${zlib:location}/lib/pkgconfig
 configure-options =
  --enable-gpl
@@ -119,5 +122,5 @@ configure-options =
 environment =
   PKG_CONFIG_PATH=${:pkg_config_depends}
   CPPFLAGS=-I${bzip2:location}/include -I${libogg:location}/include -I${libtheora:location}/include -I${opencore-amr:location}/include -I${lame:location}/include
-  LDFLAGS=-Wl,-rpath=${buildout:parts-directory}/${:_buildout_section_name_}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${libxcb:location}/lib -L${libogg:location}/lib -L${libvorbis:location}/lib -Wl,-rpath=${libvorbis:location}/lib -L${libtheora:location}/lib -Wl,-rpath=${libtheora:location}/lib -L${libvpx:location}/lib -Wl,-rpath=${libvpx:location}/lib -L${libx264:location}/lib -Wl,-rpath=${libx264:location}/lib -L${lame:location}/lib -Wl,-rpath=${lame:location}/lib -L${opencore-amr:location}/lib -Wl,-rpath=${opencore-amr:location}/lib -Wl,-rpath=${zlib:location}/lib
+  LDFLAGS=-Wl,-rpath=${:location}/lib -L${bzip2:location}/lib -Wl,-rpath=${bzip2:location}/lib -Wl,-rpath=${libxcb:location}/lib -L${libogg:location}/lib -L${libvorbis:location}/lib -Wl,-rpath=${libvorbis:location}/lib -L${libtheora:location}/lib -Wl,-rpath=${libtheora:location}/lib -L${libvpx:location}/lib -Wl,-rpath=${libvpx:location}/lib -L${libx264:location}/lib -Wl,-rpath=${libx264:location}/lib -L${lame:location}/lib -Wl,-rpath=${lame:location}/lib -L${opencore-amr:location}/lib -Wl,-rpath=${opencore-amr:location}/lib -Wl,-rpath=${zlib:location}/lib
   PATH=${pkgconfig:location}/bin:${yasm:location}/bin:%(PATH)s

component/mosquitto/buildout.cfg

+[buildout]
+extends =
+  ../cmake/buildout.cfg
+
+[mosquitto]
+recipe = slapos.recipe.cmmi
+url = https://mosquitto.org/files/source/mosquitto-2.0.15.tar.gz
+md5sum = 22b7a8b05caa692cb22496b791529193
+configure-command = 
+  ${cmake:location}/bin/cmake
+configure-options = 
+  -DWITH_CJSON=no
+  -DCMAKE_INSTALL_PREFIX=@@LOCATION@@

component/postgresql/buildout.cfg

@@ -29,7 +29,8 @@ configure-options =
   --without-libxml
   --without-libxslt
 # build core PostgreSQL + pg_trgm contrib extension for GitLab
-make-targets = install && make -C contrib/pg_trgm/ install
+# unaccent contrib extension is for peertube
+make-targets = install && make -C contrib/pg_trgm/ install && make -C contrib/unaccent/ install
 environment =
   CPPFLAGS=-I${zlib:location}/include -I${readline:location}/include -I${openssl:location}/include -I${ncurses:location}/lib
   LDFLAGS=-L${zlib:location}/lib -Wl,-rpath=${zlib:location}/lib -L${readline:location}/lib -Wl,-rpath=${readline:location}/lib -L${openssl:location}/lib -Wl,-rpath=${openssl:location}/lib -L${ncurses:location}/lib -Wl,-rpath=${ncurses:location}/lib -L${perl:location}/libs-c -Wl,-rpath=${perl:location}/libs-c

component/python3/buildout.cfg

@@ -51,23 +51,29 @@ environment =
 [python3.7]
 <= python3-common
 version = 3.7
-package_version = 3.7.13
-md5sum = 10822726f75fd7efe05a94fbd6ac2258
+package_version = 3.7.15
+md5sum = d2ff16776b5d822efc2a7cbf42fc2915
 
 [python3.8]
 <= python3-common
 version = 3.8
-package_version = 3.8.13
-md5sum = c4b7100dcaace9d33ab1fda9a3a038d6
+package_version = 3.8.15
+md5sum = cca78a827d2327f5c3ff2dee9f526e7e
 
 [python3.9]
 <= python3-common
 version = 3.9
-package_version = 3.9.13
-md5sum = 5e2411217b0060828d5f923eb422a3b8
+package_version = 3.9.15
+md5sum = 8adc5662c9fd10a23ae8ae9f28b65b49
 
 [python3.10]
 <= python3-common
 version = 3.10
-package_version = 3.10.6
-md5sum = afc7e14f7118d10d1ba95ae8e2134bf0
+package_version = 3.10.8
+md5sum = e92356b012ed4d0e09675131d39b1bde
+
+[python3.11]
+<= python3-common
+version = 3.11
+package_version = 3.11.0
+md5sum = fe92acfa0db9b9f5044958edb451d463

setup.py

@@ -121,7 +121,6 @@ setup(name=name,
           'notifier = slapos.recipe.notifier:Recipe',
           'notifier.callback = slapos.recipe.notifier:Callback',
           'notifier.notify = slapos.recipe.notifier:Notify',
-          'novnc = slapos.recipe.novnc:Recipe',
           'onetimeupload = slapos.recipe.onetimeupload:Recipe',
           'pbs = slapos.recipe.pbs:Recipe',
           'postgres = slapos.recipe.postgres:Recipe',

software/beremiz-ide/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [instance]
 filename = instance.cfg.in
-md5sum = db7fe97f4c6b84cfadb7b5c68eab517a
+md5sum = e8aae0fe3a8bc3f006b8638ed326bbcb
 
 [template-instance-beremiz]
 filename = instance-beremiz.cfg.jinja2.in
-md5sum = 9d59c9392636ab71f37b43fd8008fd10
+md5sum = 2b990148e527117bcfb366f8b700c807
 
 [template-instance-beremiz-test]
 filename = instance-beremiz-test.cfg.jinja2.in
@@ -28,3 +28,7 @@ md5sum = a2fa2b9d3a225a1dd71db67bd4fea769
 [template-fluxbox-menu.in]
 filename = fluxbox-menu.in
 md5sum = 09560314eae0225b6085f8626f1a603a
+
+[template-nginx_conf.in]
+filename = nginx_conf.in
+md5sum = 9ca886120a99befe25ca761ddc54753c

software/beremiz-ide/instance-beremiz.cfg.jinja2.in

@@ -27,35 +27,55 @@ stop-on-error = true
 cert-file = ${directory:ssl}/beremiz.crt
 key-file = ${directory:ssl}/beremiz.key
 
-[novnc-instance]
-recipe = slapos.cookbook:novnc
-path = ${directory:bin}/novnc
+[nginx-tempdir]
+recipe = slapos.cookbook:mkdirectory
+tmp = ${buildout:directory}/tmp
+client-body-temp-path = ${:tmp}/client_body_temp_path
+proxy-temp-path = ${:tmp}/proxy_temp_path
+fastcgi-temp-path = ${:tmp}/fastcgi_temp_path
+uwsgi-temp-path = ${:tmp}/uwsgi_temp_path
+scgi-temp-path = ${:tmp}/scgi_temp_path
+
+[nginx-launcher]
+recipe = slapos.cookbook:wrapper
+command-line = {{ nginx_executable }} -c ${nginx-config:output}
+wrapper-path = ${directory:services}/nginx
+
+[nginx-config]
+recipe = slapos.recipe.template:jinja2
+url = {{ template_nginx }}
+output = ${directory:etc}/nginx.conf
+context =
+    section params nginx-params
+    section ca gen-certificate
+    section tempdir nginx-tempdir
+    raw docroot {{ novnc_location }}
+    raw mime {{ nginx_mime }}
+
+[nginx-params]
+path-pid = ${directory:run}/nginx.pid
+path-error-log = ${directory:log}/nginx-error.log
+path-access-log = ${directory:log}/nginx-access.log
 ip = {{ ipv6 }}
 port = 6080
-vnc-ip = {{ ipv4 }}
-vnc-port = ${x11vnc:port}
-novnc-location = {{ novnc_location }}
-websockify-path = {{ websockify_bin }}
-ssl-key-path = ${gen-certificate:key-file}
-ssl-cert-path = ${gen-certificate:cert-file}
-
-[websockify-sighandler]
-recipe = slapos.cookbook:signalwrapper
-wrapper-path = ${directory:bin}/websockify-sighandler
-wrapped-path = ${novnc-instance:path}
-
-[websockify-sighandler-service]
-recipe = slapos.cookbook:wrapper
-command-line = ${websockify-sighandler:wrapper-path}
-wrapper-path = ${directory:services}/websockify
-hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+websocket-ip = {{ ipv4 }}
+websocket-port = ${x11vnc:port}
+websocket-path = websockify
+nb-workers = 2
+
+[nginx-graceful]
+recipe = slapos.recipe.template
+output = ${directory:scripts}/nginx-graceful
+inline =
+  #!/bin/sh
+  exec kill -s SIGHUP $(cat ${nginx-params:path-pid})
 
 [novnc-promise]
 <= monitor-promise-base
 promise = check_socket_listening
 name = novnc_promise.py
-config-host = ${novnc-instance:ip}
-config-port = ${novnc-instance:port}
+config-host = ${nginx-params:ip}
+config-port = ${nginx-params:port}
 
 [x11vnc]
 recipe = slapos.cookbook:wrapper
@@ -216,7 +236,8 @@ recipe = slapos.cookbook:requestoptional
 shared = true
 config-https-only = True
 config-type = websocket
-config-url = https://[${novnc-instance:ip}]:${novnc-instance:port}
+config-url = https://[${nginx-params:ip}]:${nginx-params:port}
+config-websocket-path-list = ${nginx-params:websocket-path}
 return = secure_access domain
 software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
 software-type = RootSoftwareInstance
@@ -226,9 +247,8 @@ name = Beremiz VNC
 [publish-connection-information]
 <= monitor-publish
 recipe = slapos.cookbook:publish
-backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?host=[${novnc-instance:ip}]&port=${novnc-instance:port}&encrypt=1
-url = ${request-vnc-frontend:connection-secure_access}/vnc.html?host=${request-vnc-frontend:connection-domain}&port=443&encrypt=1
-vnc-password = ${random-password:passwd}
+backend-url = https://[${nginx-params:ip}]:${nginx-params:port}/vnc.html?encrypt=1&password=${random-password:passwd}
+url = ${request-vnc-frontend:connection-secure_access}/vnc.html?encrypt=1&password=${random-password:passwd}
 
 [buildout]
 extends =
@@ -242,7 +262,8 @@ parts =
   xserver-promise
   x11vnc-listen-promise
   beremiz-x11
-  websockify-sighandler-service
+  nginx-launcher
+  nginx-graceful
   request-vnc-frontend
   generate-vnc-password
   publish-connection-information

software/beremiz-ide/instance.cfg.in

@@ -34,6 +34,7 @@ context =
   key computer_id              slap-configuration:computer
   raw bin_directory            {{ bin_directory }}
   raw template_monitor         {{ template_monitor_cfg }}
+  raw template_nginx           {{ template_nginx }}
   raw template_logrotate       {{ logrotate_cfg }}
   raw logrotate_cfg            {{ logrotate_cfg }}
   raw python_bin               {{ python_bin }}
@@ -62,9 +63,10 @@ extra-context =
   raw gtk3_location      {{ gtk3_location }}
   raw matiec_location    {{ matiec_location }}
   raw mesa_location      {{ mesa_location }}
+  raw nginx_executable   {{ nginx_executable }}
+  raw nginx_mime         {{ nginx_mime }}
   raw novnc_location     {{ novnc_location }}
   raw openssl_bin        {{ openssl_location }}/bin/openssl
-  raw websockify_bin     {{ bin_directory }}/websockify
   raw x11vnc_bin         {{ x11vnc_location }}/bin/x11vnc
   raw xvfb_bin           {{ xserver_location }}/bin/Xvfb
   raw xterm_bin          {{ xterm_location }}/bin/xterm

software/beremiz-ide/nginx_conf.in

+worker_processes {{ params['nb-workers'] }};
+
+pid {{ params['path-pid'] }};
+error_log {{ params['path-error-log'] }};
+
+daemon off;
+
+events {
+  worker_connections 1024;
+  accept_mutex off;
+}
+
+http {
+    include {{ mime }};
+    default_type  application/octet-stream;
+    types_hash_bucket_size 64;
+    access_log {{ params['path-access-log'] }} combined;
+    index novnc.html;
+
+    upstream vnc_proxy {
+        server {{ params['websocket-ip'] }}:{{ params['websocket-port'] }};
+    }
+    server {
+        listen [{{ params['ip'] }}]:{{ params['port'] }} ssl http2;
+        server_name _;
+
+        ssl_certificate {{ ca['cert-file'] }};
+        ssl_certificate_key {{ ca['key-file'] }};
+        ssl_session_timeout 1d;
+        ssl_session_cache shared:MozSSL:10m;
+        ssl_session_tickets off;
+        ssl_protocols TLSv1.3;
+        ssl_prefer_server_ciphers off;
+
+        keepalive_timeout 5;
+        client_body_temp_path {{ tempdir['client-body-temp-path'] }};
+        proxy_temp_path {{ tempdir['proxy-temp-path'] }};
+        fastcgi_temp_path {{ tempdir['fastcgi-temp-path'] }};
+        uwsgi_temp_path {{ tempdir['uwsgi-temp-path'] }};
+        scgi_temp_path {{ tempdir['scgi-temp-path'] }};
+
+        # path for static files
+        root {{ docroot }};
+
+        location /{{ params['websocket-path'] }} {
+            proxy_http_version 1.1;
+            proxy_pass http://vnc_proxy/;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            # VNC connection timeout
+            proxy_read_timeout 61s;
+
+            # Disable cache
+            proxy_buffering off;
+        }
+
+    }
+}

software/beremiz-ide/software.cfg

@@ -9,7 +9,6 @@ extends =
   ../../component/noVNC/buildout.cfg
   ../../component/nginx/buildout.cfg
   ../../component/lxml-python/buildout.cfg
-  ../../component/numpy/buildout.cfg
   ../../component/numpy/openblas.cfg
   ../../component/matplotlib/buildout.cfg
   ../../component/wxpython/buildout.cfg
@@ -74,10 +73,8 @@ recipe = zc.recipe.egg
 eggs =
   ${wxPython:egg}
   ${python-cryptography:egg}
-  ${lxml-python:egg}
   ${matplotlib:egg}
   future
-  websockify
   zeroconf2
   enum34
   pyro
@@ -131,6 +128,7 @@ context =
     key template_instance_beremiz template-instance-beremiz:target
     key template_instance_beremiz_test template-instance-beremiz-test:target
     key template_logrotate template-logrotate-base:output
+    key template_nginx template-nginx_conf.in:target
     key fontconfig_location fontconfig:location
     key font_dejavu dejavu-fonts:location
     key font_liberation liberation-fonts:location
@@ -144,6 +142,8 @@ context =
     key instance_template_type :type
     key matiec_location matiec:location
     key mesa_location mesa:location
+    key nginx_executable nginx-output:nginx
+    key nginx_mime nginx-output:mime
     key novnc_location noVNC:location
     key nxdtest_template nxdtest-instance.cfg:output
     key python_bin :python-bin
@@ -168,12 +168,15 @@ output = ${buildout:directory}/instance-beremiz-test.cfg.jinja2
 <= download-template
 output = ${buildout:directory}/fluxbox-menu.in
 
+[template-nginx_conf.in]
+<= download-template
+output = ${buildout:directory}/nginx_conf.in
+
 [versions]
 Pillow = 6.2.2
 matplotlib = 2.2.5
 kiwisolver = 1.1.0
 cycler = 0.10.0
-websockify = 0.9.0
 Pyro = 3.16
 zeroconf2 = 0.19.2
 cython = 0.29.24

software/caddy-frontend/software.cfg

@@ -10,7 +10,6 @@ extends =
   ../../component/6tunnel/buildout.cfg
   ../../component/xz-utils/buildout.cfg
   ../../component/rsyslogd/buildout.cfg
-  ../../component/numpy/buildout.cfg
   ../../component/haproxy/buildout.cfg
   ../../component/nginx/buildout.cfg
   ../../component/findutils/buildout.cfg
@@ -57,7 +56,6 @@ depends = ${software-develop:recipe}
 recipe = zc.recipe.egg
 eggs =
   software
-  websockify
 
 [profile-common]
 recipe = slapos.recipe.template:jinja2
@@ -226,6 +224,5 @@ plone.recipe.command = 1.1
 pycrypto = 2.6.1
 smmap = 0.9.0
 
-websockify = 0.8.0
 furl = 2.1.0
 orderedmultidict = 1.0.1

software/kvm/buildout.hash.cfg

@@ -15,11 +15,11 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = a7978940fb9cdcc4e1ec33015ba640ba
+md5sum = b6204319cca4264b3c351d4dd1f2b5d0
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 69749ef4be49b970af9548d68e6d8785
+md5sum = 14a8433ca9f0038bb6cc4b68ef7ea8e5
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
@@ -49,13 +49,17 @@ md5sum = 64aa1ce8785f6b94aabd787fa3443082
 filename = instance-nbd.cfg.jinja2
 md5sum = e041e8011ad2ec7f104be173ef76f5e9
 
+[template-nginx]
+filename = template/nginx_conf.in
+md5sum = 9ca886120a99befe25ca761ddc54753c
+
 [template-ansible-promise]
 filename = template/ansible-promise.in
 md5sum = 6328f99728284847b8dd1146aadeae1b
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = fa048a28da7362d570f5b6bd1e05d232
+md5sum = 4ce3fc8072e1e010ee99651cb01d3b3d
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm.cfg.jinja2

@@ -643,31 +643,48 @@ promise = check_command_execute
 name = qemu-virtual-machine-is-ready.py
 config-command = ${kvm-started-bin:output}
 
-[novnc-instance]
-recipe = slapos.cookbook:novnc
-path = ${ca-novnc:executable}
+[nginx-tempdir]
+recipe = slapos.cookbook:mkdirectory
+tmp = ${buildout:directory}/tmp
+client-body-temp-path = ${:tmp}/client_body_temp_path
+proxy-temp-path = ${:tmp}/proxy_temp_path
+fastcgi-temp-path = ${:tmp}/fastcgi_temp_path
+uwsgi-temp-path = ${:tmp}/uwsgi_temp_path
+scgi-temp-path = ${:tmp}/scgi_temp_path
+
+[nginx-launcher]
+recipe = slapos.cookbook:wrapper
+command-line = ${ca-novnc:executable} -c ${nginx-config:output}
+wrapper-path = ${directory:services}/nginx
+
+[nginx-config]
+recipe = slapos.recipe.template:jinja2
+url = {{ template_nginx }}
+output = ${directory:etc}/nginx.conf
+context =
+    section params nginx-params
+    section ca ca-novnc
+    section tempdir nginx-tempdir
+    raw docroot {{ novnc_location }}
+    raw mime {{ nginx_mime }}
+
+[nginx-params]
+path-pid = ${directory:run}/nginx.pid
+path-error-log = ${directory:log}/nginx-error.log
+path-access-log = ${directory:log}/nginx-access.log
 ip = ${slap-network-information:global-ipv6}
 port = 6080
-vnc-ip = ${kvm-parameter-dict:vnc-ip}
-vnc-port = ${kvm-parameter-dict:vnc-port}
-novnc-location = {{ novnc_location }}
-websockify-path = {{ websockify_executable_location }}
-ssl-key-path = ${ca-novnc:key-file}
-ssl-cert-path = ${ca-novnc:cert-file}
-
-[websockify-sighandler]
-recipe = slapos.cookbook:signalwrapper
-wrapper-path = ${directory:bin}/websockify-sighandler
-wrapped-path = ${novnc-instance:path}
-
-[websockify-sighandler-service]
-recipe = slapos.cookbook:wrapper
-command-line = ${websockify-sighandler:wrapper-path}
-wrapper-path = ${directory:services}/websockify
-hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
-wait-for-files =
-  ${ca-novnc:key-file}
-  ${ca-novnc:cert-file}
+websocket-ip = ${kvm-parameter-dict:vnc-ip}
+websocket-port = ${kvm-parameter-dict:vnc-port}
+websocket-path = websockify
+nb-workers = 2
+
+[nginx-graceful]
+recipe = slapos.recipe.template
+output = ${directory:scripts}/nginx-graceful
+inline =
+  #!/bin/sh
+  exec kill -s SIGHUP $(cat ${nginx-params:path-pid})
 
 [certificate-authority]
 recipe = slapos.cookbook:certificate_authority
@@ -699,15 +716,15 @@ crl = ${directory:ca-dir}/crl/
 recipe = slapos.cookbook:certificate_authority.request
 key-file = ${directory:novnc-conf}/novnc.key
 cert-file = ${directory:novnc-conf}/novnc.crt
-executable = ${directory:bin}/novnc
-wrapper = ${directory:bin}/websockify
+executable = {{ nginx_executable }}
+wrapper = ${directory:bin}/nginx-with-ca
 
 [novnc-promise]
 <= monitor-promise-base
 promise = check_socket_listening
 name = novnc_promise.py
-config-host = ${novnc-instance:ip}
-config-port = ${novnc-instance:port}
+config-host = ${nginx-params:ip}
+config-port = ${nginx-params:port}
 
 
 #----------------
@@ -748,7 +765,8 @@ partition-id = ${slap-connection:partition-id}
 shared = true
 config-https-only = True
 config-type = websocket
-config-url = https://[${novnc-instance:ip}]:${novnc-instance:port}
+config-websocket-path-list = ${nginx-params:websocket-path}
+config-url = https://[${nginx-params:ip}]:${nginx-params:port}
 return = secure_access domain
 
 [request-slave-frontend]
@@ -762,7 +780,7 @@ sla-instance_guid = ${slap-parameter:frontend-instance-guid}
 <= monitor-promise-base
 promise = check_url_available
 name = frontend_promise.py
-config-url = ${request-slave-frontend:connection-secure_access}
+config-url = ${request-slave-frontend:connection-secure_access}/vnc.html
 
 {% if additional_frontend %}
 [request-slave-frontend-additional]
@@ -823,10 +841,10 @@ blank-line =
 <= monitor-publish
 recipe = slapos.cookbook:publish.serialised
 ipv6 = ${slap-network-information:global-ipv6}
-backend-url = https://[${novnc-instance:ip}]:${novnc-instance:port}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
-url = ${request-slave-frontend:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
+backend-url = https://[${nginx-params:ip}]:${nginx-params:port}/vnc.html?encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
+url = ${request-slave-frontend:connection-secure_access}/vnc.html?encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
 {% if additional_frontend %}
-url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?auto=1&encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
+url-additional = ${request-slave-frontend-additional:connection-secure_access}/vnc.html?encrypt=1&password=${kvm-controller-parameter-dict:vnc-passwd}
 {% endif %}
 {% set disk_number = len(storage_dict) -%}
 maximum-extra-disk-amount = {{ disk_number }}
@@ -1242,8 +1260,8 @@ parts =
   kvm-controller-wrapper
   kvm-vnc-promise
   kvm-disk-image-corruption-promise
-  websockify-sighandler
-  websockify-sighandler-service
+  nginx-launcher
+  nginx-graceful
   novnc-promise
   kvm-started-promise
   cron

software/kvm/instance.cfg.in

@@ -89,6 +89,8 @@ extra-context =
   raw logrotate_cfg ${template-logrotate-base:output}
   raw novnc_location ${noVNC:location}
   raw netcat_bin ${netcat:location}/bin/netcat
+  raw nginx_executable ${nginx-output:nginx}
+  raw nginx_mime ${nginx-output:mime}
   raw python_executable ${buildout:executable}
   raw python_eggs_executable ${buildout:bin-directory}/${python-with-eggs:interpreter}
   raw qemu_executable_location ${qemu:location}/bin/qemu-system-x86_64
@@ -100,6 +102,7 @@ extra-context =
   raw template_kvm_controller_run ${template-kvm-controller:target}
   raw template_kvm_run ${template-kvm-run:target}
   raw template_monitor ${monitor2-template:output}
+  raw template_nginx ${template-nginx:target}
   raw websockify_executable_location ${buildout:directory}/bin/websockify
   raw wipe_disk_wrapper ${buildout:directory}/bin/securedelete
 template-parts-destination = ${template-parts:target}

software/kvm/software.cfg

@@ -8,9 +8,8 @@ extends =
   ../../component/noVNC/buildout.cfg
   ../../component/openssl/buildout.cfg
   ../../component/netcat/buildout.cfg
-  ../../component/lxml-python/buildout.cfg
+  ../../component/nginx/buildout.cfg
   ../../component/pycurl/buildout.cfg
-  ../../component/numpy/buildout.cfg
   ../../component/gzip/buildout.cfg
   ../../stack/slapos.cfg
   ../../stack/resilient/buildout.cfg
@@ -33,28 +32,20 @@ parts = ${:common-parts}
 #             In qemu builtin vnc server, and make it available only for localhost
 #             so that only novnc can listen to it.
 
-#XXX-Cedric: Check status of https://github.com/kanaka/noVNC/issues/13 to see
-#            When qemu has builtin support for websockets in vnc server to get rid of
-#            Websockify (socket <-> websocket proxy server) when it is ready.
-#            May solve previous XXX depending on the implementation.
-
-#XXX-Cedric : add list of keyboard layouts (azerty/us querty/...) parameter to qemu
-
 [python-with-eggs]
 recipe = zc.recipe.egg
 interpreter = ${:_buildout_section_name_}
 eggs =
   ${slapos-toolbox:eggs}
   ${python-cffi:egg}
-  ${lxml-python:egg}
-  websockify
   ${slapos-cookbook:eggs}
   erp5.util
 # BBB: eggs used as recipe should be kept otherwise sections depending
 # on it can't be uninstalled
   collective.recipe.shelloutput
-scripts =
-  websockify
+# Only generate the interpreter script to avoid conflicts with scripts
+# for eggs that are also generated by another section, like slapos.toolbox
+scripts = ${:interpreter}
 
 # Create all templates that will be used to deploy instances
 [download-base]
@@ -97,6 +88,9 @@ output = ${buildout:directory}/template.cfg
 [template-nbd]
 <= download-base
 
+[template-nginx]
+<= download-base
+
 [template-ansible-promise]
 <= download-base
 
@@ -135,9 +129,3 @@ context =
 
 [whitelist-domains-default]
 <= download-base
-
-[versions]
-websockify = 0.9.0
-gitdb = 0.6.4
-pycurl = 7.43.0
-smmap = 0.9.0

software/kvm/template/nginx_conf.in

+worker_processes {{ params['nb-workers'] }};
+
+pid {{ params['path-pid'] }};
+error_log {{ params['path-error-log'] }};
+
+daemon off;
+
+events {
+  worker_connections 1024;
+  accept_mutex off;
+}
+
+http {
+    include {{ mime }};
+    default_type  application/octet-stream;
+    types_hash_bucket_size 64;
+    access_log {{ params['path-access-log'] }} combined;
+    index novnc.html;
+
+    upstream vnc_proxy {
+        server {{ params['websocket-ip'] }}:{{ params['websocket-port'] }};
+    }
+    server {
+        listen [{{ params['ip'] }}]:{{ params['port'] }} ssl http2;
+        server_name _;
+
+        ssl_certificate {{ ca['cert-file'] }};
+        ssl_certificate_key {{ ca['key-file'] }};
+        ssl_session_timeout 1d;
+        ssl_session_cache shared:MozSSL:10m;
+        ssl_session_tickets off;
+        ssl_protocols TLSv1.3;
+        ssl_prefer_server_ciphers off;
+
+        keepalive_timeout 5;
+        client_body_temp_path {{ tempdir['client-body-temp-path'] }};
+        proxy_temp_path {{ tempdir['proxy-temp-path'] }};
+        fastcgi_temp_path {{ tempdir['fastcgi-temp-path'] }};
+        uwsgi_temp_path {{ tempdir['uwsgi-temp-path'] }};
+        scgi_temp_path {{ tempdir['scgi-temp-path'] }};
+
+        # path for static files
+        root {{ docroot }};
+
+        location /{{ params['websocket-path'] }} {
+            proxy_http_version 1.1;
+            proxy_pass http://vnc_proxy/;
+            proxy_set_header Upgrade $http_upgrade;
+            proxy_set_header Connection "upgrade";
+
+            # VNC connection timeout
+            proxy_read_timeout 61s;
+
+            # Disable cache
+            proxy_buffering off;
+        }
+
+    }
+}

software/kvm/template/template-kvm-run.in

@@ -277,7 +277,7 @@ ram = '%sM,slots=128,maxmem=%sM' % (init_ram_size, ram_max_size)
 
 kvm_argument_list = [qemu_path,
   '-enable-kvm', '-smp', smp, '-name', vm_name, '-m', ram, '-vga', 'std',
-  '-vnc', '%s:1,ipv4=on,password=on' % listen_ip,
+  '-vnc', '%s:1,password=on,websocket=on' % listen_ip,
   '-boot', 'order=cd,menu=on',
   '-qmp', 'unix:%s,server,nowait' % socket_path,
   '-pidfile', pid_file_path, '-msg', 'timestamp=on',

software/kvm/test/test.py

@@ -212,7 +212,8 @@ i0:kvm-{kvm-hash-value}-on-watch RUNNING
 i0:kvm_controller EXITED
 i0:monitor-httpd-{hash}-on-watch RUNNING
 i0:monitor-httpd-graceful EXITED
-i0:websockify-{hash}-on-watch RUNNING
+i0:nginx-graceful EXITED
+i0:nginx-on-watch RUNNING
 i0:whitelist-domains-download-{hash} RUNNING
 i0:whitelist-firewall-{hash} RUNNING""",
       self.getProcessInfo()
@@ -702,11 +703,12 @@ ir2:kvm-{kvm-hash-value}-on-watch RUNNING
 ir2:kvm_controller EXITED
 ir2:monitor-httpd-{hash}-on-watch RUNNING
 ir2:monitor-httpd-graceful EXITED
+ir2:nginx-graceful EXITED
+ir2:nginx-on-watch RUNNING
 ir2:notifier-on-watch RUNNING
 ir2:resilient_sshkeys_authority-on-watch RUNNING
 ir2:sshd-graceful EXITED
 ir2:sshd-on-watch RUNNING
-ir2:websockify-{hash}-on-watch RUNNING
 ir2:whitelist-domains-download-{hash} RUNNING
 ir2:whitelist-firewall-{hash} RUNNING
 ir3:bootstrap-monitor EXITED
@@ -2295,7 +2297,8 @@ ihs0:kvm-{kvm-hash-value}-on-watch RUNNING
 ihs0:kvm_controller EXITED
 ihs0:monitor-httpd-{hash}-on-watch RUNNING
 ihs0:monitor-httpd-graceful EXITED
-ihs0:websockify-{hash}-on-watch RUNNING
+ihs0:nginx-graceful EXITED
+ihs0:nginx-on-watch RUNNING
 ihs0:whitelist-domains-download-{hash} RUNNING
 ihs0:whitelist-firewall-{hash} RUNNING""",
       self.getProcessInfo()

software/metabase/instance.cfg.in

@@ -70,7 +70,6 @@ config-url= $${metabase-instance:url}/api/session/properties
 
 [metabase-keystore-password]
 recipe = slapos.cookbook:generate.password
-bytes = 24
 
 [metabase-keystore]
 recipe = plone.recipe.command

software/mosquitto/buildout.hash.cfg

+[instance-profile]
+filename = instance.cfg.in
+md5sum = 4c7aa7b2132dc13ddee37fb416decf81

software/mosquitto/instance.cfg.in

+[buildout]
+parts =
+  promises
+  mosquitto-service
+  publish-connection-parameter
+
+eggs-directory = {{ buildout["eggs-directory"] }}
+develop-eggs-directory = {{ buildout["develop-eggs-directory"] }}
+offline = true
+
+[instance-parameter]
+recipe = slapos.cookbook:slapconfiguration
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
+[check-port-listening-promise]
+recipe = slapos.cookbook:check_port_listening
+path = ${directory:promise}/${:_buildout_section_name_}
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+home = ${buildout:directory}
+etc = ${:home}/etc
+var = ${:home}/var
+script = ${:etc}/run/
+service = ${:etc}/service
+promise = ${:etc}/promise/
+log = ${:var}/log
+bin = ${:home}/bin
+
+[mosquitto-config-file]
+recipe = slapos.recipe.build
+location = ${directory:etc}/${:_buildout_section_name_}.cfg
+ipv4 = ${instance-parameter:ipv4-random}
+ipv6 = ${instance-parameter:ipv6-random}
+port = 1883
+password = ${mosquitto-password-file:location}
+install =
+  config = open(self.options["location"], "w")
+  port = self.options["port"]
+  ipv4 = self.options["ipv4"]
+  ipv6 = self.options["ipv6"]
+  password = self.options["password"]
+  config.write(f"listener {port} {ipv4}\nprotocol mqtt\n\n")
+  config.write(f"listener {port} {ipv6}\nprotocol mqtt\n\n")
+  config.write(f"password_file {password}\n\n")
+
+[mosquitto-password-file]
+recipe = plone.recipe.command
+location = ${directory:etc}/${:_buildout_section_name_}.txt
+command = 
+  touch ${:location}
+  {{ mosquitto_location }}/bin/mosquitto_passwd -b ${:location} ${mosquitto-password:username} ${mosquitto-password:passwd}
+stop-on-error = true
+
+[mosquitto-password]
+recipe = slapos.cookbook:generate.password
+username = mosquitto
+
+[mosquitto-listen-promise-ipv4]
+<= check-port-listening-promise
+hostname = ${mosquitto-config-file:ipv4}
+port = ${mosquitto-config-file:port}
+
+[mosquitto-listen-promise-ipv6]
+<= check-port-listening-promise
+hostname = ${mosquitto-config-file:ipv6}
+port = ${mosquitto-config-file:port}
+
+[promises]
+recipe =
+instance-promises =
+  ${mosquitto-listen-promise-ipv4:path}
+  ${mosquitto-listen-promise-ipv6:path}
+
+[mosquitto-service]
+recipe  = slapos.cookbook:wrapper
+command-line = {{ mosquitto_location }}/sbin/mosquitto -c ${mosquitto-config-file:location}
+wrapper-path = ${directory:service}/mosquitto-service
+output = $${:wrapper-path}
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+ipv4 = mqtt://${mosquitto-config-file:ipv4}:${mosquitto-config-file:port}
+ipv6 = mqtt://${mosquitto-config-file:ipv6}:${mosquitto-config-file:port}
+username = ${mosquitto-password:username}
+password = ${mosquitto-password:passwd}

software/mosquitto/software.cfg

+[buildout]
+extends =
+  buildout.hash.cfg
+  ../../stack/slapos.cfg
+  ../../stack/monitor/buildout.cfg
+  ../../component/mosquitto/buildout.cfg
+
+parts =
+  slapos-cookbook
+  instance-profile
+  plone.recipe.command
+
+[plone.recipe.command]
+recipe = zc.recipe.egg
+
+[instance-profile]
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/instance.cfg.in
+rendered = ${buildout:directory}/instance.cfg
+context =
+  section buildout buildout
+  key mosquitto_location mosquitto:location

software/peertube/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+[instance-profile]
+filename = instance.cfg.in
+md5sum = 35690065ba18dc49d0108fc1f0a07b9e
+
+[instance-peertube]
+_update_hash_filename_ = instance-peertube.cfg.in
+md5sum = 52b9a872d69052f53dd29bfa4a067dbf
+
+[template-nginx-service]
+filename = template-nginx-service.sh.in
+md5sum = 458870b70c33a1621b68961ae2372ad5
+
+[template-peertube-service]
+filename = template-peertube-service.sh.in
+md5sum = fe42401ea8df7a245955683535f8a063
+
+[template-peertube-yaml]
+filename = template-peertube.yaml.in
+md5sum = e761995c4c18efc4a199f11dc8fde039
+
+[template-nginx-configuration]
+filename = template-nginx.cfg.in
+md5sum = a2c0d6e873370d64de707f668a3d40e3
+
+[template-dcron-service]
+filename = template-dcron-service.sh.in
+md5sum = 851262d7174da868805cb7c8e1ced7c0
+
+[template-crontab-line]
+filename = template-crontab-line.in
+md5sum = bab861f1d057e837697d78e74e47b0bb
+
+[template-crontab]
+filename = template-crontab.in
+md5sum = f1f82101258de19068262b7213fc478b
+
+[template-peertube-backup-script]
+filename = template-peertube-backup.sh.in
+md5sum = e50daa16a2c1866997933981bed45271
+
+[template-peertube-restore-script]
+filename = template-peertube-restore.sh.in
+md5sum = bb67602f3d652d4e5a06711fae5996f9

software/peertube/instance-peertube-input-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "type": "object",
+  "description": "Parameters to instantiate Peertube",
+  "additionalProperties": false,
+  "properties": {
+    "frequency": {
+      "title": "Backup Frequency",
+      "description": "Backup Frequency",
+      "default": "",
+      "type": "string"
+    },
+    "fronted-url": {
+      "title": "Frontend URL",
+      "description": "Frontend URL",
+      "default": "",
+      "type": "string"
+    },
+    "name": {
+      "title": "Instance Title",
+      "description": "The name of your peertube instance.",
+      "default": "Peertube in Slapos",
+      "type": "string"
+    },
+    "short_description": {
+      "title": "PeerTube Short Description",
+      "description": "Short description of your peertube instance.",
+      "default": "PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.",
+      "type": "string"
+    },
+    "description": {
+      "title": "Peertube Long Description",
+      "description": "The description of your instance",
+      "default": "Welcome to this PeerTube instance!",
+      "textarea": true,
+      "type": "string"
+    },
+    "terms": {
+      "title": "Terms",
+      "description": "",
+      "default": "No terms for now.",
+      "textarea": true,
+      "type": "string"
+    },
+    "code_of_conduct": {
+      "title": "Code of Conduct",
+      "description": "",
+      "default": "",
+      "textarea": true,
+      "type": "string"
+    },
+    "moderation_information": {
+      "title": "Moderation Information",
+      "description": "Who moderates the instance? What is the policy regarding NSFW videos? Political videos? etc",
+      "default": "",
+      "textarea": true,
+      "type": "string"
+    },
+    "creation_reason": {
+      "title": "Creation Reason",
+      "description": "Why did you create this instance?",
+      "default": "",
+      "textarea": true,
+      "type": "string"
+    },
+    "administrator": {
+      "title":"Administrator",
+      "description": "Who is behind the instance? A single person? A non profit?",
+      "default": "",
+      "type": "string"
+    },
+    "maintenance_lifetime": {
+      "title": "Maintenance Lifetime",
+      "description": "How long do you plan to maintain this instance?",
+      "default": "",
+      "type": "string"
+    },
+    "business_model": {
+      "title": "Business Model",
+      "description": "How will you pay the PeerTube instance server? With your own funds? With users donations? Advertising?",
+      "default": "",
+      "textarea": true,
+      "type": "string"
+    },
+    "hardware_information": {
+      "title": "Hardware Information",
+      "description": "",
+      "default": "",
+      "textarea": true,
+      "type": "string"
+    }
+  }
+}

software/peertube/instance-peertube-output-schema.json

+{
+  "$schema": "http://json-schema.org/draft-04/schema#",
+  "description": "Values returned by Peertube instanciation",
+  "properties": {
+    "frontend-url": {
+      "description": "Peertube Frontend URL",
+      "type": "string"
+    },
+    "backend-url": {
+      "description": "Peertube Backend URL",
+      "type": "string"
+    },
+    "username": {
+      "description": "Peertube username",
+      "type": "string"
+    },
+    "password": {
+      "description": "Peertube password",
+      "type": "string"
+    }
+  },
+  "type": "object"
+}

software/peertube/instance-peertube.cfg.in

+[buildout]
+extends =
+  {{ monitor_template }}
+
+parts =
+  service-redis
+  promise-redis
+  postgresql
+  postgresql-binary-link
+  nginx-service
+  nginx-listen-promise
+  peertube-yaml
+  peertube-service
+  peertube-listen-promise
+  dcron-service
+  activate-crontab-file
+  peertube-backup-cron
+  peertube-database-resiliency-after-import-script
+  peertube-database-resiliency-exclude-file
+  publish-connection-parameter
+
+eggs-directory = {{ eggs_directory }}
+develop-eggs-directory = {{ develop_eggs_directory }}
+offline = true
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = ${buildout:directory}/etc
+srv = ${buildout:directory}/srv
+var = ${buildout:directory}/var
+log = ${:var}/log
+run = ${:var}/run
+www = ${:var}/www
+crontabs = ${:etc}/crontabs
+cron-entries = ${:etc}/cron.d
+cronstamps = ${:etc}/cronstamps
+cron-lines = ${:etc}/cron.lines
+peertube_nginx_log = ${:log}/nginx
+varnginx = ${:var}/nginx
+services = ${:etc}/service
+peertube_directory = ${:www}/peertube
+config = ${:peertube_directory}/config
+storage = ${:peertube_directory}/storage
+versions = ${:peertube_directory}/versions
+ssl = ${:etc}/ssl
+
+##################
+#   Postgresql   #
+##################
+
+[postgresql-password]
+recipe = slapos.cookbook:generate.password
+
+[postgresql-address]
+recipe = slapos.cookbook:free_port
+minimum = 5432
+maximum = 5452
+ip = {{ ipv4_random }}
+
+[postgresql]
+recipe = slapos.cookbook:postgres
+bin = {{ postgresql10_location }}/bin/
+services = ${directory:services}
+dbname  = peertube_prod
+superuser = peertube
+password = ${postgresql-password:passwd}
+pgdata-directory = ${directory:srv}/postgresql
+
+ipv4 = ${postgresql-address:ip}
+# disable listening on ipv6
+ipv6 =
+port = ${postgresql-address:port}
+
+[postgresql-binary-link]
+recipe = slapos.cookbook:symbolic.link
+target-directory = ${directory:bin}
+link-binary = ${postgresql:bin}/postgres ${postgresql:bin}/psql
+
+#############
+#   Nginx   #
+#############
+
+[nginx-service]
+recipe = slapos.recipe.template
+url = {{ template_nginx_service }}
+output = ${directory:services}/nginx
+virtual-depends =
+  ${nginx-configuration:ip}
+
+[nginx-listen-promise]
+<= monitor-promise-base
+promise = check_url_available
+name = nginx_listen.py
+config-verify = 0
+config-url = https://[${nginx-configuration:ip}]:${nginx-configuration:port}
+
+[nginx-configuration]
+recipe = slapos.recipe.template
+url = {{ template_nginx_configration }}
+output = ${directory:etc}/nginx.cfg
+access_log = ${directory:log}/nginx-access.log
+error_log = ${directory:log}/nginx-error.log
+ip = {{ ipv6_random }}
+port = 9443
+ssl_key = ${directory:ssl}/nginx.key
+ssl_csr = ${directory:ssl}/nginx.csr
+ssl_crt = ${directory:ssl}/nginx.crt
+
+#############
+#   Redis   #
+#############
+[redis]
+recipe  = slapos.cookbook:mkdirectory
+srv     = ${directory:srv}/redis
+log     = ${directory:log}/redis
+
+[service-redis]
+recipe  = slapos.cookbook:redis.server
+wrapper = ${directory:services}/redis
+promise-wrapper = ${directory:bin}/redis-promise
+
+server-dir  = ${redis:srv}
+config-file = ${directory:etc}/redis.conf
+log-file    = ${redis:log}/redis.log
+pid-file    = ${directory:run}/redis.pid
+use-passwd  = false
+unixsocket  = ${:server-dir}/redis.socket
+# port = 0 means "don't listen on TCP at all" - listen only on unix socket
+ipv6    = ::1
+port    = 0
+
+server-bin = {{ redis_binprefix }}/redis-server
+cli-bin  = {{ redis_binprefix }}/redis-cli
+depend  =
+    ${logrotate-entry-redis:recipe}
+
+[promise-redis]
+<= monitor-promise-base
+promise = check_command_execute
+name = promise-redis.py
+config-command = ${service-redis:promise-wrapper}
+
+[logrotate-entry-redis]
+<= logrotate-entry-base
+log     = ${redis:log}/*.log
+name = redis
+
+################
+#   Peertube   #
+################
+
+[peertube-passwd]
+recipe = slapos.cookbook:generate.password
+username = root
+
+[peertube-yaml]
+recipe = slapos.recipe.template
+url = {{ template_peertube_yaml }}
+output = ${directory:config}/peertube.yaml
+
+[peertube-listen-promise]
+<= monitor-promise-base
+promise = check_url_available
+name = peertube_listen.py
+config-verify = 0
+config-url = ${peertube-parameters:frontend-url}
+
+[peertube-service]
+recipe = slapos.recipe.template
+url = {{ template_peertube_service }}
+output = ${directory:services}/peertube
+
+[peertube-database-resiliency-exclude-file]
+recipe = slapos.recipe.template:jinja2
+inline = {{ "${postgresql:pgdata-directory}/" }}
+output = ${directory:srv}/exporter.exclude
+
+[peertube-database-resiliency-after-import-script]
+recipe = slapos.recipe.template
+url = {{ template_peertube_restore }}
+output = ${directory:srv}/runner-import-restore
+mode = 755
+
+#################################
+#         Cron service          #
+#################################
+[dcron-service]
+recipe = slapos.recipe.template
+url = {{ template_dcron_service }}
+output = ${directory:services}/crond
+logfile = ${directory:log}/crond.log
+
+[peertube-backup-script]
+recipe = slapos.recipe.template
+url = {{ template_peertube_backup }}
+backup-file = ${directory:srv}/backup/peertube_prod-dump.db
+output = ${directory:bin}/${:_buildout_section_name_}
+mode = 0744
+
+[peertube-backup-cron]
+recipe = slapos.recipe.template
+url = {{ template_crontab_line }}
+output = ${directory:bin}/${:_buildout_section_name_}
+script = ${peertube-backup-script:output}
+name = ${:_buildout_section_name_}
+frequency = {{ slapparameter_dict.get('frequency', '@daily') }}
+
+[activate-crontab-file]
+# XXX File is never removed
+recipe = plone.recipe.command
+stop-on-error = true
+command = {{ coreutils_cat }} {{ template_crontab }} ${peertube-backup-cron:output} | {{ dcron_output }} -c ${directory:crontabs} -
+
+[frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = PeerTube Server Frontend
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+shared = true
+{% if slapparameter_dict.get('frontend-url', 'false') != 'false' -%}
+state = stopped
+{% endif -%}
+config-type = websocket
+config-websocket-path-list = /socket.io /socket /tracker/socket
+config-url = https://[${nginx-configuration:ip}]:${nginx-configuration:port}
+return = domain secure_access
+
+[peertube-parameters]
+recipe = slapos.recipe.build
+slapparameter-dict = {{ dumps(slapparameter_dict) }}
+default-frontend-url = ${frontend:connection-secure_access}
+working-dir = {{ peertube_location }}
+ipv4-port=9000
+node-config-dir=${directory:config}
+node-env=production
+npm-bin={{ nodejs_location }}/bin/npm
+default-parameters =
+  {
+    "name" : "Peertube in Slapos",
+    "frontend-url" : "",
+    "short_description" : "PeerTube, an ActivityPub-federated video streaming platform using P2P directly in your web browser.",
+    "description" : "Welcome to this PeerTube instance!",
+    "terms" : "No terms for now.",
+    "code_of_conduct" : "",
+    "moderation_information" : "",
+    "creation_reason" : "",
+    "administrator" : "",
+    "maintenance_lifetime" : "",
+    "business_model" : "",
+    "hardware_information" : ""
+  }
+init =
+  from six.moves.urllib.parse import urlparse
+  import json
+  default_parameters = json.loads(options.get('default-parameters').strip())
+  default_parameters['frontend-url'] = options['default-frontend-url']
+  parameters = dict(default_parameters, **options['slapparameter-dict'])
+  # options is not a dict...
+  options['name'] = parameters['name']
+  options['frontend-url'] = parameters['frontend-url']
+  options['short_description'] = parameters['short_description']
+  options['description'] = parameters['description']
+  options['terms'] = parameters['terms']
+  options['code_of_conduct'] = parameters['code_of_conduct']
+  options['moderation_information'] = parameters['moderation_information']
+  options['creation_reason'] = parameters['creation_reason']
+  options['administrator'] = parameters['administrator']
+  options['maintenance_lifetime'] = parameters['maintenance_lifetime']
+  options['business_model'] = parameters['business_model']
+  options['hardware_information'] = parameters['hardware_information']
+  url = urlparse(options.get('frontend-url'))
+  if url.port:
+    options['host'] = '[' + url.hostname + ']'
+    options['port'] = str(url.port)
+  else:
+    options['host'] = url.hostname
+    options['port'] = str(443)
+
+[publish-connection-parameter]
+recipe = slapos.cookbook:publish
+backend-url = ${frontend:config-url}
+frontend-url = ${peertube-parameters:frontend-url}
+password = ${peertube-passwd:passwd}
+username = ${peertube-passwd:username}

software/peertube/instance.cfg.in

+[buildout]
+parts =
+  switch-softwaretype
+
+eggs-directory = ${buildout:eggs-directory}
+develop-eggs-directory = ${buildout:develop-eggs-directory}
+offline = true
+
+[peertube]
+recipe = slapos.recipe.template:jinja2
+url = ${instance-peertube:target}
+output = $${buildout:directory}/instance-peertube.cfg
+context =
+  key eggs_directory              buildout:eggs-directory
+  key develop_eggs_directory      buildout:develop-eggs-directory
+  raw monitor_template            ${monitor2-template:output}
+  key slapparameter_dict          slap-configuration:configuration
+  raw ipv6_random                 $${slap-configuration:ipv6-random}
+  raw ipv4_random                 $${slap-configuration:ipv4-random}
+  raw template_peertube_yaml      ${template-peertube-yaml:output}
+  raw template_nginx_configration ${template-nginx-configuration:output}
+  raw template_peertube_service   ${template-peertube-service:output}
+  raw template_peertube_restore   ${template-peertube-restore-script:output}
+  raw postgresql10_location       ${postgresql10:location}
+  raw template_nginx_service      ${template-nginx-service:output}
+  raw redis_binprefix             ${redis28:location}/bin
+  raw template_dcron_service      ${template-dcron-service:output}
+  raw template_peertube_backup    ${template-peertube-backup-script:output}
+  raw template_crontab_line       ${template-crontab-line:output}
+  raw coreutils_cat               ${coreutils-output:cat}
+  raw template_crontab            ${template-crontab:output}
+  raw dcron_output                ${dcron-output:crontab}
+  raw peertube_location           ${peertube:location}
+  raw nodejs_location             ${nodejs:location}
+
+[switch-softwaretype]
+recipe = slapos.cookbook:switch-softwaretype
+RootSoftwareInstance = $${:default}
+default = peertube:output
+
+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer= $${slap-connection:computer-id}
+partition=$${slap-connection:partition-id}
+url     = $${slap-connection:server-url}
+key     = $${slap-connection:key-file}
+cert    = $${slap-connection:cert-file}
+

software/peertube/software.cfg

+[buildout]
+
+extends =
+  buildout.hash.cfg
+# buildout.hash.cfg is used for automated hash calculation of managed
+# instance files by calling update-hash
+# buildout.hash.cfg
+# "slapos" stack describes basic things needed for 99.9% of SlapOS Software
+# Releases
+  ../../stack/slapos.cfg
+  ../../stack/monitor/buildout.cfg
+# Extend here component profiles, like openssl, apache, mariadb, curl...
+# Or/and extend a stack (lamp, tomcat) that does most of the work for you
+# In this example we extend from helloweb component.
+  ../../component/unzip/buildout.cfg
+  ../../component/curl/buildout.cfg
+  ../../component/dcron/buildout.cfg
+  ../../component/vim/buildout.cfg
+  ../../component/nodejs/buildout.cfg
+  ../../component/yarn/buildout.cfg
+  ../../component/python3/buildout.cfg
+  ../../component/nginx/buildout.cfg
+  ../../component/ffmpeg/buildout.cfg
+  ../../component/postgresql/buildout.cfg
+  ../../component/nspr/buildout.cfg
+  ../../component/gcc/buildout.cfg
+  ../../component/libxml2/buildout.cfg
+  ../../component/libxslt/buildout.cfg
+  ../../component/lxml-python/buildout.cfg
+# redis-server
+  ../../component/redis/buildout.cfg
+  ../../component/git/buildout.cfg
+  ../../component/wget/buildout.cfg
+# backup
+  ../../component/rdiff-backup/buildout.cfg
+  ../../component/rsync/buildout.cfg
+
+parts =
+# Call installation of slapos.cookbook egg defined in stack/slapos.cfg (needed
+# in 99,9% of Slapos Software Releases)
+  slapos-cookbook
+  dcron
+  gcc
+  unzip
+  curl
+  nodejs
+  yarn
+  openssl
+  python3
+  nginx
+  ffmpeg
+  postgresql
+  redis
+  wget
+# peetube sections
+  peertube
+  peertube-build
+  instance-profile
+
+[nodejs]
+<= nodejs-16.13.2
+
+[peertube]
+recipe = slapos.recipe.build:download-unpacked
+url = https://github.com/Chocobozzz/PeerTube/releases/download/v4.2.2/peertube-v4.2.2.zip
+md5sum = 1c9639748d66e8c49fc27e4705f87622
+
+[peertube-build]
+recipe = slapos.recipe.cmmi
+path = ${peertube:location}
+environment =
+  PATH=${unzip:location}/bin:${vim:location}/bin:${nodejs:location}/bin:${yarn:location}/bin:${python3:location}/bin:${nginx:location}/sbin:${postgresql10:location}/bin:${gcc-10.2:location}/bin:${redis:location}/bin:{git:location}/bin:{wget:location}/bin:%(PATH)s
+  CPPFLAGS=-I${openssl:location}/include
+  LDFLAGS=-L${curl:location}/lib -Wl,-rpath -Wl,${openssl:location}/lib -Wl
+pre-configure =
+  ${yarn:location}/bin/yarn install --production --pure-lockfile
+configure-command = true
+make-binary = cd ${peertube:location} && ${yarn:location}/bin/yarn
+post-install =
+  rm -rf ${buildout:directory}/.cache/yarn/
+  rm -rf ${buildout:directory}/parts/peertube/client/node_modules/chromedriver/
+
+[instance-profile]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/instance.cfg
+
+[instance-peertube]
+recipe  = slapos.recipe.build:download
+url     = ${:_profile_base_location_}/${:_update_hash_filename_}
+destination = ${buildout:directory}/${:_buildout_section_name_}
+
+[template-peertube-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+
+[template-peertube-yaml]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+
+[template-nginx-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+
+[template-nginx-configuration]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+
+[template-dcron-service]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+
+[template-crontab-line]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+
+[template-crontab]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+
+[template-peertube-backup-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}
+
+[template-peertube-restore-script]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/${:filename}

software/peertube/software.cfg.json

+{
+  "name": "Peertube",
+  "description": "Peertube Website Text",
+  "serialisation": "json-in-xml",
+  "software-type": {
+    "default": {
+      "title": "Default",
+      "software-type": "default",
+      "description": "Default",
+      "request": "instance-peertube-input-schema.json",
+      "response": "instance-peertube-output-schema.json",
+      "index": 1
+    }
+  }
+}

software/peertube/template-crontab-line.in

+# $${:_buildout_section_name_}
+$${:frequency} ID=$${:name} $${:script}

software/peertube/template-crontab.in

+# min(0-59)  hours(0-23) day(1-31) month(1-12) dow(0-7)   command
+MAILTO=admins@erp5.org

software/peertube/template-dcron-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapos node
+# BEWARE: It will be overwritten automatically
+
+exec ${dcron-output:crond} \
+  -s $${directory:cron-entries} \
+  -c $${directory:crontabs} \
+  -t $${directory:cronstamps} \
+  -f -l 5 \
+  -L $${dcron-service:logfile}
+  # -M cron_simplelogger

software/peertube/template-nginx-service.sh.in

+#!${dash-output:dash}
+# BEWARE: This file is operated by slapos node
+# BEWARE: It will be overwritten automatically
+
+if [ ! -e $${nginx-configuration:ssl_crt} ]
+then
+  ${openssl-output:openssl} genrsa -out $${nginx-configuration:ssl_key} 2048
+  ${openssl-output:openssl} req -new \
+    -subj "/C=AA/ST=Denial/L=Nowhere/O=Dis/CN=$${nginx-configuration:ip}" \
+    -key $${nginx-configuration:ssl_key} -out $${nginx-configuration:ssl_csr}
+  ${openssl-output:openssl} x509 -req -days 365 \
+    -in $${nginx-configuration:ssl_csr} \
+    -signkey $${nginx-configuration:ssl_key} \
+    -out $${nginx-configuration:ssl_crt}
+fi
+
+exec ${nginx-output:nginx} \
+  -c $${nginx-configuration:output}

software/peertube/template-peertube-backup.sh.in

+#!${dash-output:dash}
+
+# Dump the databse
+$${postgresql:bin}/pg_dump -h $${postgresql:pgdata-directory} -U $${postgresql:superuser} -Fc peertube_prod > $${peertube-backup-script:backup-file}

software/peertube/template-peertube-restore.sh.in

+#!${dash-output:dash}
+
+# DO NOT RUN THIS SCRIPT ON PRODUCTION INSTANCE
+# OR POSTGRESQL DATA WILL BE ERASED.
+
+# This script will import the dump of the postgresql database to the real
+# database. It is launched by the clone (importer) instance of theia
+# in the end of the import script.
+
+# Depending on the output, it will create a file containing
+# the status of the restoration (success or failure)
+
+die() {
+    echo "$*" 1>&2
+    exit 1
+}
+
+echo "Postgresql data directory is ready"
+# 2. Make sure the postgresql process is not running.
+# Quote from the postgresql doc:
+# > While the server is running, its PID is stored in the file postmaster.pid in the data directory.
+# https://www.postgresql.org/docs/current/server-start.html
+# which means if the postmaster.pid exist, then the postgresql is running.
+pid_file=$${postgresql:pgdata-directory}/postmaster.pid
+if [ -e "$pid_file" ]; then
+  echo "Postgresql is running, this should not happened, aborting."
+  exit 1
+fi
+
+echo "Starting postgresql..."
+$${postgresql:bin}/postgres -D $${postgresql:pgdata-directory} &
+postgresql_pid=$!
+trap "kill $postgresql_pid" EXIT TERM INT
+
+# If postgres has stopped, abort
+if ! [ -d /proc/$postgresql_pid ]; then
+  echo "postgresql exited, aborting."
+  exit 1
+fi
+
+# run psql
+psql() {
+    PGPASSWORD=$${postgresql:password} $${postgresql:bin}/psql \
+        -h $${postgresql:ipv4}              \
+        -p $${postgresql:port}              \
+        -U $${postgresql:superuser}         \
+        -d $${postgresql:dbname}            \
+        "$@"
+}
+
+echo "Ready to check postgresql is running..."
+
+# initial db setup
+# ( first quering PG several times waiting a bit till postgresql is started and ready )
+tpgwait=60
+while true; do
+    pgtables="$(psql -c '\d' 2>&1)" && break
+    tpgwait=$(( $tpgwait - 1 ))
+    test $tpgwait = 0 && die "pg query problem"
+    echo "I: PostgreSQL is not ready (yet ?); will retry $tpgwait times..." 1>&2
+    sleep 1
+done
+echo "I: PostgreSQL ready." 1>&2
+
+echo "Postgresql is running, ready to restore"
+
+# Restore the database
+# Use -d to connect to the default 'postgres' database to allow us to restore the $${postgresql:dbname}
+PGPASSWORD=$${postgresql:password} $${postgresql:bin}/pg_restore -h $${postgresql:ipv4} -p $${postgresql:port} -U $${postgresql:superuser} -e -c -C -d postgres $${peertube-backup-script:backup-file} || {
+  RESTORE_EXIT_CODE=$?
+  echo 'Backup restoration failed.'
+  exit $RESTORE_EXIT_CODE
+}
+
+echo "Postgresql restore finished"

software/peertube/template-peertube-service.sh.in

+#!${dash-output:dash}
+
+# setup db
+
+die() {
+    echo "$*" 1>&2
+    exit 1
+}
+
+# run psql on gitlab db
+psql() {
+    PGPASSWORD=$${postgresql:password} $${postgresql:bin}/psql \
+        -h $${postgresql:ipv4}              \
+        -p $${postgresql:port}              \
+        -U $${postgresql:superuser}         \
+        -d $${postgresql:dbname}            \
+        "$@"
+}
+
+# initial db setup
+# ( first quering PG several times waiting a bit till postgresql is started and ready )
+tpgwait=5
+while true; do
+    pgtables="$(psql -c '\d' 2>&1)" && break
+    tpgwait=$(( $tpgwait - 1 ))
+    test $tpgwait = 0 && die "pg query problem"
+    echo "I: PostgreSQL is not ready (yet ?); will retry $tpgwait times..." 1>&2
+    sleep 1
+done
+echo "I: PostgreSQL ready." 1>&2
+
+cd ${peertube:location}
+
+exec env PT_INITIAL_ROOT_PASSWORD=$${peertube-passwd:passwd} NODE_ENV=peertube\
+  NODE_CONFIG_DIR=$${directory:config} PATH=${ffmpeg:location}/bin:$PATH\
+  ${nodejs:location}/bin/node ${peertube:location}/dist/server

software/peertube/test/README.md

+Tests for Peertube software release

slapos/recipe/novnc/__init__.py → software/peertube/test/setup.py

 ##############################################################################
 #
-# Copyright (c) 2011 Vifib SARL and Contributors. All Rights Reserved.
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
 #
 # WARNING: This program as such is intended to be used by professional
 # programmers who take the whole responsibility of assessing all potential
@@ -24,29 +24,29 @@
 # Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
 #
 ##############################################################################
-from slapos.recipe.librecipe import GenericBaseRecipe
-import binascii
-import os
-import sys
+from setuptools import setup, find_packages
 
-class Recipe(GenericBaseRecipe):
-  """
-  novnc instance configuration.
-  """
+version = '0.0.1.dev0'
+name = 'slapos.test.peertube'
+with open("README.md") as f:
+  long_description = f.read()
 
-  def install(self):
-    return self.createWrapper(
-      self.options['path'],
-      (
-        self.options['websockify-path'],
-        '--web',
-        self.options['novnc-location'],
-        '--key=%s' % self.options['ssl-key-path'],
-        '--cert=%s' % self.options['ssl-cert-path'],
-        '--ssl-only',
-        '%s:%s' % (self.options['ip'], self.options['port']),
-        '%s:%s' % (self.options['vnc-ip'], self.options['vnc-port']),
-       ),
-      wait_list=(self.options['ssl-key-path'],
-                 self.options['ssl-cert-path']),
-    )
+setup(
+    name=name,
+    version=version,
+    description="Test for SlapOS' peertube",
+    long_description=long_description,
+    long_description_content_type='text/markdown',
+    maintainer="Nexedi",
+    maintainer_email="info@nexedi.com",
+    url="https://lab.nexedi.com/nexedi/slapos",
+    packages=find_packages(),
+    install_requires=[
+        'slapos.core',
+        'slapos.libnetworkcache',
+        'erp5.util',
+        'requests',
+    ],
+    zip_safe=True,
+    test_suite='test',
+)

software/peertube/test/test.py

+##############################################################################
+#
+# Copyright (c) 2019 Nexedi SA and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import os
+import requests
+import re
+from mimetypes import guess_type
+from json.decoder import JSONDecodeError
+
+from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
+
+setUpModule, SlapOSInstanceTestCase = makeModuleSetUpAndTestCaseClass(
+    os.path.abspath(
+        os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+
+
+class TestPeerTube(SlapOSInstanceTestCase):
+  def setUp(self):
+    self.connection_parameters = self.computer_partition.getConnectionParameterDict()
+
+  def test_get(self):
+    url = self.connection_parameters['frontend-url']
+    response = requests.get(url, verify=False)
+    self.assertEqual(requests.codes['OK'], response.status_code)
+    self.assertIn('PeerTube', response.text)
+    response =requests.get(url + "/feeds/videos.xml?sort=-trending", verify=False)
+    self.assertEqual(requests.codes['OK'], response.status_code)
+    self.assertIn('rss', response.text)
+
+  def test_video_upload(self):
+    api_url = self.connection_parameters['frontend-url']
+    # api_url: https://[2001:67c:1254:fd::9ee2]:9443
+    # self.connection_parameters
+    # {'backend-url': 'https://[2001:67c:1254:fd::9ee2]:9443', 'frontend-hostname': '[2001:67c:1254:fd::9ee2]:9443', 'frontend-url': 'https://[2001:67c:1254:fd::9ee2]:9443', 'password': '8ydTfRpv', 'username': 'root'}
+    response = requests.get(api_url + '/api/v1/oauth-clients/local', verify=False)
+    self.assertEqual(requests.codes['OK'], response.status_code)
+    try:
+      data = response.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube API is incorrect.")
+
+    client_id = data['client_id']
+    client_secret = data['client_secret']
+    username = self.connection_parameters['username']
+    password = self.connection_parameters['password']
+    auth_data = {
+        'client_id': client_id,
+        'client_secret': client_secret,
+        'grant_type': 'password',
+        'response_type': 'code',
+        'username': username,
+        'password': password
+    }
+
+    auth_result = requests.post(api_url + '/api/v1/users/token', data=auth_data, verify=False)
+    self.assertEqual(requests.codes['OK'], auth_result.status_code)
+    try:
+      auth_result_json = auth_result.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube API is incorrect.")
+
+    token_type = auth_result_json['token_type']
+    access_token = auth_result_json['access_token']
+    headers = {
+        'Authorization': token_type + ' ' + access_token
+    }
+    video_name = "Small test video"
+    file_path = "./small.mp4"
+    file_mime_type = guess_type(file_path)[0]
+
+    with open(file_path, 'rb') as f:
+        video_data = {
+            'channelId': 1,
+            'name': video_name,
+            'commentEnabled': False,
+        }
+        upload_response = requests.post(
+            api_url + '/api/v1/videos/upload',
+            headers=headers,
+            data=video_data,
+            files={'videofile': (os.path.basename(file_path), f, file_mime_type)},
+            verify=False
+        )
+    try:
+      video_ids = upload_response.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube API is incorrect.")
+    # {'video': {'id': 7, 'shortUUID': 'nrnKJNCsRP7NkwRr51TK3e', 'uuid': 'ad9ae99d-07db-4e4c-adc3-73566d59a4c5'}}
+    self.assertIn('video', video_ids)
+    id = video_ids['video']['id']
+
+    # Check the video is uploaded, we can get its stats
+    response = requests.get(api_url + '/api/v1/videos/' + str(id) + '/stats/overall', headers=headers, verify=False)
+    self.assertEqual(requests.codes['OK'], response.status_code)
+    try:
+      result = response.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube API is incorrect.")
+    self.assertIn('totalWatchTime', response.json())
+
+    # Check the transcoding is enabled
+    response = requests.get(api_url + '/api/v1/config', headers=headers, verify=False)
+    try:
+      result = response.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube API is incorrect.")
+    # {
+    #   'hls': {'enabled': True},
+    #   'webtorrent': {'enabled': False},
+    #   'enabledResolutions': [144, 240, 360, 480, 720, 1080, 1440, 2160],
+    #   'profile': 'default',
+    #   'availableProfiles': ['default']
+    # }
+    self.assertIn("hls", result['transcoding'])
+    self.assertIn("True", str(result['transcoding']['hls']))

software/slapos-sr-testing/software.cfg

@@ -242,6 +242,11 @@ setup = ${slapos-repository:location}/software/erp5testnode/test/
 egg = slapos.test.beremiz_ide
 setup = ${slapos-repository:location}/software/beremiz-ide/test/
 
+[slapos.test.peertube-setup]
+<= setup-develop-egg
+egg = slapos.test.peertube
+setup = ${slapos-repository:location}/software/peertube/test/
+
 [slapos.core-repository]
 <= git-clone-repository
 repository = https://lab.nexedi.com/nexedi/slapos.core.git
@@ -316,6 +321,7 @@ eggs +=
   ${slapos.test.nextcloud-setup:egg}
   ${slapos.test.nginx-push-stream-setup:egg}
   ${slapos.test.ors-amarisoft-setup:egg}
+  ${slapos.test.peertube-setup:egg}
   ${slapos.test.plantuml-setup:egg}
   ${slapos.test.powerdns-setup:egg}
   ${slapos.test.proftpd-setup:egg}
@@ -406,6 +412,7 @@ tests =
   nextcloud ${slapos.test.nextcloud-setup:setup}
   nginx-push-stream ${slapos.test.nginx-push-stream-setup:setup}
   ors-amarisoft ${slapos.test.ors-amarisoft-setup:setup}
+  peertube ${slapos.test.peertube-setup:setup}
   plantuml ${slapos.test.plantuml-setup:setup}
   powerdns ${slapos.test.powerdns-setup:setup}
   proftpd ${slapos.test.proftpd-setup:setup}

software/theia/test/project_tests.py

@@ -38,6 +38,8 @@ import requests
 
 from datetime import datetime, timedelta
 from six.moves.urllib.parse import urljoin
+from mimetypes import guess_type
+from json.decoder import JSONDecodeError
 
 from slapos.testing.testcase import installSoftwareUrlList
 
@@ -48,6 +50,9 @@ from test import SlapOSInstanceTestCase, theia_software_release_url
 erp5_software_release_url = os.path.abspath(
   os.path.join(
     os.path.dirname(__file__), '..', '..', 'erp5', 'software.cfg'))
+peertube_software_release_url = os.path.abspath(
+  os.path.join(
+    os.path.dirname(__file__), '..', '..', 'peertube', 'software.cfg'))
 
 
 def setUpModule():
@@ -232,3 +237,209 @@ class TestTheiaResilienceERP5(ERP5Mixin, test_resiliency.TestTheiaResilience):
     # Check that the mariadb catalog was properly restored
     out = subprocess.check_output((mysql_bin, 'erp5', '-e', query), universal_newlines=True)
     self.assertIn(self._erp5_new_title, out, 'Mariadb catalog is not properly restored')
+
+class TestTheiaResiliencePeertube(test_resiliency.TestTheiaResilience):
+  test_instance_max_retries = 12
+  backup_max_tries = 480
+  backup_wait_interval = 60
+  _connexion_parameters_regex = re.compile(r"{.*}", re.DOTALL)
+  _test_software_url = peertube_software_release_url
+
+  def _getPeertubeConnexionParameters(self, instance_type='export'):
+    out = self.captureSlapos(
+      'request', 'test_instance', self._test_software_url,
+      stderr=subprocess.STDOUT,
+      text=True,
+    )
+    print(out)
+    return json.loads(self._connexion_parameters_regex.search(out).group(0).replace("'", '"'))
+
+  def test_twice(self):
+    # do nothing
+    pass
+
+  def _prepareExport(self):
+    super(TestTheiaResiliencePeertube, self)._prepareExport()
+
+    postgresql_partition = self._getPeertubePartitionPath('export', 'postgres')
+    postgresql_bin = os.path.join(postgresql_partition, 'bin', 'psql')
+    postgres_bin = os.path.join(postgresql_partition, 'bin', 'postgres')
+    postgresql_srv = os.path.join(postgresql_partition, 'srv', 'postgresql')
+
+    peertube_conenction_info = self._getPeertubeConnexionParameters()
+    frontend_url = peertube_conenction_info['frontend-url']
+
+    response = requests.get(frontend_url + '/api/v1/oauth-clients/local', verify=False)
+    self.assertEqual(requests.codes['OK'], response.status_code)
+    try:
+      data = response.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube API is incorrect.")
+
+    client_id = data['client_id']
+    client_secret = data['client_secret']
+    username = peertube_conenction_info['username']
+    password = peertube_conenction_info['password']
+    auth_data = {
+        'client_id': client_id,
+        'client_secret': client_secret,
+        'grant_type': 'password',
+        'response_type': 'code',
+        'username': username,
+        'password': password
+    }
+
+    auth_result = requests.post(frontend_url + '/api/v1/users/token', data=auth_data, verify=False)
+    try:
+      auth_result_json = auth_result.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube API is incorrect.")
+
+    token_type = auth_result_json['token_type']
+    access_token = auth_result_json['access_token']
+    headers = {
+        'Authorization': token_type + ' ' + access_token
+    }
+    video_name = "Small test video"
+    file_path = "../../peertube/test/small.mp4"
+    pwd_file_path = os.path.realpath(__file__)
+    print(pwd_file_path)
+    file_mime_type = guess_type(file_path)[0]
+
+    with open(file_path, 'rb') as f:
+        video_data = {
+            'channelId': 1,
+            'name': video_name,
+            'commentEnabled': False,
+            'privacy': 1,
+        }
+        upload_response = requests.post(
+            frontend_url + '/api/v1/videos/upload',
+            headers=headers,
+            data=video_data,
+            files={'videofile': (os.path.basename(file_path), f, file_mime_type)},
+            verify=False
+        )
+    try:
+      video_ids = upload_response.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube API is incorrect.")
+    # e.g: {'video': {'id': 7, 'shortUUID': 'nrnKJNCsRP7NkwRr51TK3e', 'uuid': 'ad9ae99d-07db-4e4c-adc3-73566d59a4c5'}}
+    self.assertIn('video', video_ids)
+
+    # Checked the modification has been updated in the database
+    output = subprocess.check_output(
+      (postgresql_bin, '-h', postgresql_srv, '-U', 'peertube', '-d', 'peertube_prod',
+      '-c', 'SELECT * FROM "video"'),
+      universal_newlines=True)
+    self.assertIn("Small test video", output)
+
+    # Do a fake periodically update
+    # Compute backup date in the near future
+    soon = (datetime.now() + timedelta(minutes=4)).replace(second=0)
+    frequency = "%d * * * *" % soon.minute
+    params = 'frequency=%s' % frequency
+
+    # Update Peertube parameters
+    print('Requesting Peertube with parameters %s' % params)
+    self.checkSlapos('request', 'test_instance', self._test_software_url, '--parameters', params)
+
+    self.checkSlapos('node', 'instance')
+
+    self.callSlapos('node', 'restart', 'all')
+
+    # Wait until after the programmed backup date, and a bit more
+    t = (soon - datetime.now()).total_seconds()
+    self.assertLess(0, t)
+    time.sleep(t + 120)
+    self.callSlapos('node', 'status')
+
+    # Check that postgresql backup has started
+    postgresql_backup = os.path.join(postgresql_partition, 'srv', 'backup')
+    self.assertIn('peertube_prod-dump.db', os.listdir(postgresql_backup))
+
+  def _checkTakeover(self):
+    super(TestTheiaResiliencePeertube, self)._checkTakeover()
+
+    postgresql_partition = self._getPeertubePartitionPath('export', 'postgres')
+    postgresql_bin = os.path.join(postgresql_partition, 'bin', 'psql')
+    postgres_bin = os.path.join(postgresql_partition, 'bin', 'postgres')
+    postgresql_srv = os.path.join(postgresql_partition, 'srv', 'postgresql')
+
+    peertube_conenction_info = self._getPeertubeConnexionParameters()
+    frontend_url = peertube_conenction_info['frontend-url']
+    storage_path = os.path.join(postgresql_partition, 'var', 'www', 'peertube', 'storage')
+
+    # Wait for connect Peertube
+    for _ in range(5):
+      try:
+        response = requests.get(frontend_url, verify=False, allow_redirects=False)
+      except Exception:
+        time.sleep(20)
+        continue
+      if response.status_code != 200:
+        time.sleep(20)
+        continue
+      break
+    else:
+      self.fail('Failed to connect to Peertube')
+
+    # Get the video path, the part of this path will be used in the video URL
+    # e.g: var/www/peertube/storage/streaming-playlists/hls/XXXX/YYYY.mp4
+
+    # path before hls dir
+    hls_path = os.path.join(storage_path, 'streaming-playlists', 'hls')
+
+    #Choose only one video path
+    video_path = None
+    for root, dirs, files in os.walk(hls_path):
+      for a_file in files:
+        if a_file.endswith('.mp4'):
+          video_path = os.path.join(root, a_file)
+          break
+      else:
+        continue
+      break
+
+    # path like "streaming-playlists/hls/XXXX/YYYY.mp4"
+    self.assertIn('streaming-playlists', video_path)
+
+    streaming_video_path = video_path[video_path.index('streaming-playlists'):]
+    video_url = frontend_url + '/static/' + streaming_video_path
+    response = requests.get(video_url, verify=False)
+    # The video mp4 file is accesible through the URL
+    self.assertEqual(requests.codes['OK'], response.status_code)
+
+    video_feeds_url = frontend_url + '/feeds/videos.json'
+    response = requests.get(video_feeds_url, verify=False)
+
+    # The video feeds returns the correct status code
+    self.assertEqual(requests.codes['OK'], response.status_code)
+    try:
+      video_data= response.json()
+    except JSONDecodeError:
+      self.fail("No json file returned! Maybe your Peertube feeds URL is incorrect.")
+
+    # Check the first video title is in the response content
+    video_title = video_data['items'][0]['title']
+    self.assertIn("Small test video", video_title)
+
+  def _getPeertubePartition(self, servicename):
+    p = subprocess.Popen(
+      (self._getSlapos(), 'node', 'status'),
+      stdout=subprocess.PIPE, universal_newlines=True)
+    out, _ = p.communicate()
+    found = set()
+    for line in out.splitlines():
+      if servicename in line:
+        found.add(line.split(':')[0])
+    if not found:
+      raise Exception("Peertube %s partition not found" % servicename)
+    elif len(found) > 1:
+      raise Exception("Found several partitions for Peertube %s" % servicename)
+    return found.pop()
+
+  def _getPeertubePartitionPath(self, instance_type, servicename, *paths):
+    partition = self._getPeertubePartition(servicename)
+    return self.getPartitionPath(
+      instance_type, 'srv', 'runner', 'instance', partition, *paths)

stack/slapos.cfg

@@ -136,18 +136,19 @@ zc.buildout = 2.7.1+slapos019
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 
-traitlets = 4.3.3
-Jinja2 = 2.11.3
-Importing = 1.10
-MarkupSafe = 2.0.1
-PyYAML = 5.4.1
-Werkzeug = 2.0.2
-ZConfig = 2.9.3
+apache-libcloud = 2.4.0
 asn1crypto = 1.3.0
 atomicwrites = 1.4.0
+atomize = 0.2.0
+attrs = 22.1.0
 backports.functools-lru-cache = 1.6.1:whl
 backports.lzma = 0.0.14
+bcrypt = 3.1.4
+CacheControl = 0.12.6:whl
+certifi = 2022.6.15
 cffi = 1.14.0
+chardet = 3.0.4
+charset-normalizer = 2.1.1
 click = 8.1.3
 cliff = 2.8.3:whl
 cmd2 = 0.7.0
@@ -155,44 +156,75 @@ collective.recipe.shelloutput = 0.1
 collective.recipe.template = 2.0
 configparser = 4.0.2:whl
 contextlib2 = 0.6.0.post1
+croniter = 0.3.25
 cryptography = 3.3.2
 dateparser = 0.7.6
 decorator = 4.3.0
+distro = 1.7.0
+dnspython = 1.16.0
+enum34 = 1.1.10
+erp5.util = 0.4.74
+feedparser = 5.2.1
+Flask = 1.1.2
 funcsigs = 1.0.2
+functools32 = 3.2.3.post2
 gevent = 20.9.0
 geventmp = 0.0.1
+gitdb2 = 2.0.5
+GitPython = 2.1.11
 greenlet = 0.4.17
 idna = 2.9
 igmp = 1.0.4
+Importing = 1.10
 importlib-metadata = 1.7.0:whl
 inotify-simple = 1.1.1
+ipaddress = 1.0.23
 itsdangerous = 0.24
+Jinja2 = 2.11.3
+jsonschema = 3.0.2:whl
 lock-file = 2.0
+lockfile = 0.12.2:whl
 lxml = 4.9.1
+MarkupSafe = 2.0.1
 meld3 = 1.0.2
 mock = 3.0.5
 more-itertools = 5.0.0
+msgpack = 0.6.2
 netaddr = 0.7.19
+netifaces = 0.10.7
+packaging = 16.8
+paramiko = 2.11.0
+passlib = 1.7.1
 pathlib2 = 2.3.5
 pbr = 2.0.0
 pim-dm = 1.4.0nxd001
+pkgconfig = 1.5.1
 plone.recipe.command = 1.1
+pluggy = 0.13.1:whl
 prettytable = 0.7.2
 psutil = 5.8.0
-pluggy = 0.13.1:whl
 py = 1.11.0:whl
+py-mld = 1.0.3
+pyasn1 = 0.4.5
+pycparser = 2.20
+pycurl = 7.43.0
+PyNaCl = 1.3.0
 pyOpenSSL = 19.1.0
 pyparsing = 3.0.9:whl
-py-mld = 1.0.3
 pyroute2 = 0.6.9
+pyrsistent = 0.18.1
+PyRSS2Gen = 1.1
+pytest-runner = 5.2:whl
+python-dateutil = 2.7.3:whl
 pytz = 2022.2.1
+PyYAML = 5.4.1
 regex = 2020.9.27
 requests = 2.28.1
-charset-normalizer = 2.1.1
+rpdb = 0.1.5
+rubygemsrecipe  = 0.4.3
 scandir = 1.10.0
 setproctitle = 1.1.10
 setuptools-dso = 1.7
-rubygemsrecipe  = 0.4.3
 six = 1.16.0
 slapos.cookbook = 1.0.291
 slapos.core = 1.8.4
@@ -203,70 +235,38 @@ slapos.recipe.build = 0.56
 slapos.recipe.cmmi = 0.19
 slapos.recipe.template = 5.0
 slapos.toolbox = 0.128
+smmap2 = 2.0.5
 stevedore = 1.21.0:whl
 subprocess32 = 3.5.4
+supervisor = 4.1.0
+traitlets = 4.3.3
+tzlocal = 1.5.1
 unicodecsv = 0.14.1
+uritemplate = 3.0.0
+urllib3 = 1.26.12
 wcwidth = 0.2.5
+Werkzeug = 2.0.2
 wheel = 0.35.1:whl
 xml-marshaller = 1.0.2
 zc.lockfile = 1.4
+ZConfig = 2.9.3
 zdaemon = 4.2.0
 zipp = 1.2.0:whl
 zodburi = 2.5.0
 zope.event = 3.5.2
-paramiko = 2.11.0
-PyNaCl = 1.3.0
-bcrypt = 3.1.4
-CacheControl = 0.12.6:whl
-msgpack = 0.6.2
-Flask = 1.1.2
-GitPython = 2.1.11
-gitdb2 = 2.0.5
-smmap2 = 2.0.5
-PyRSS2Gen = 1.1
-apache-libcloud = 2.4.0
-atomize = 0.2.0
-croniter = 0.3.25
-dnspython = 1.16.0
-enum34 = 1.1.10
-erp5.util = 0.4.74
-feedparser = 5.2.1
-functools32 = 3.2.3.post2
-attrs = 22.1.0
-pyrsistent = 0.18.1
-pytest-runner = 5.2:whl
-ipaddress = 1.0.23
-jsonschema = 3.0.2:whl
-lockfile = 0.12.2:whl
-netifaces = 0.10.7
-packaging = 16.8
-passlib = 1.7.1
-pyasn1 = 0.4.5
-pycparser = 2.20
-pycurl = 7.43.0
-python-dateutil = 2.7.3:whl
-rpdb = 0.1.5
-supervisor = 4.1.0
-tzlocal = 1.5.1
-uritemplate = 3.0.0
 zope.interface = 5.4.0
-certifi = 2022.6.15
-chardet = 3.0.4
-urllib3 = 1.26.12
-pkgconfig = 1.5.1
-distro = 1.7.0
 
 
 [versions:python2]
 attrs = 18.2.0
+certifi = 2020.4.5.1
+charset-normalizer = 2.0.12
 click = 6.7
 distro = 1.6.0
-Werkzeug = 1.0.1
-requests = 2.27.1
-charset-normalizer = 2.0.12
 pyparsing = 2.2.0
-certifi = 2020.4.5.1
 pyrsistent = 0.16.1
+requests = 2.27.1
+Werkzeug = 1.0.1
 
 [versions:sys.version_info < (3,8)]
 MarkupSafe = 1.0