caddy-frontend: Implement type:websocket

By default whole slave makes websocket connection to the backend.
With websocket-path, only the path has websocket style connections,
the rest is standard HTTP.

software/caddy-frontend/buildout.hash.cfg

@@ -26,11 +26,11 @@ md5sum = bde0f62dfe2eeef8f10b4315535095cb
 
 [template-apache-replicate]
 filename = instance-apache-replicate.cfg.in
-md5sum = a4303904fa1dfebcbb40f28cd715e7cf
+md5sum = d62aefe002ec13875924e4c219914795
 
 [template-slave-list]
 filename = templates/apache-custom-slave-list.cfg.in
-md5sum = 71dfc1c57988416f5a40ced83acda2a7
+md5sum = 75439cb035393e68c73672b224bead54
 
 [template-slave-configuration]
 filename = templates/custom-virtualhost.conf.in
@@ -54,7 +54,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = 9de6875635038f88be4f039e03deb1c0
+md5sum = 0c5ef7f26a142c3ab53e835d2caa698d
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in

software/caddy-frontend/instance-apache-replicate.cfg.in

@@ -82,7 +82,7 @@ context =
 {%   set slave_type = slave.get('type') %}
 {%   if slave_type == 'eventsource' %}
 {%     do slave_error_list.append('type:eventsource is not implemented') %}
-{%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook'] %}
+{%   elif slave_type not in [None, '', 'default', 'zope', 'redirect', 'notebook', 'websocket'] %}
 {%     do slave_error_list.append('type:%s is not supported' % (slave_type,)) %}
 {%   endif %}
 {#   BBB: apache_custom_https AND apache_custom_http #}

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -107,6 +107,22 @@
       "title": "type:zope Backend Path",
       "type": "string"
     },
+    "websocket-path-list": {
+      "default": "",
+      "description": "Space separated list of path to the websocket application. If not set the whole slave will be websocket, if set then / will be HTTP, and /<websocket-path> will be WSS. In order to have ' ' in the space use '%20'",
+      "title": "type:websocket Websocket Application Path List",
+      "type": "string"
+    },
+    "websocket-transparent": {
+      "default": "true",
+      "description": "If set to false, websocket slave will be without Caddy's transparent proxy mode. Depending on the application the setting shall be false or true. Defaults to true for transparent proxying.",
+      "enum": [
+        "false",
+        "true"
+      ],
+      "title": "type:websocket Transparent proxy",
+      "type": "string"
+    },
     "prefer-gzip-encoding-to-backend": {
       "default": "false",
       "description": "If set to true, frontend will rewrite Accept-Encoding request header to simply 'gzip' for all variants of Accept-Encoding containing 'gzip', in order to maximize cache hits for resources cached with Vary: Accept-Encoding when enable_cache is used",
@@ -148,12 +164,13 @@
     },
     "type": {
       "default": "",
-      "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect and notebook, not implemneted is eventsource.",
+      "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
       "enum": [
         "",
         "zope",
         "redirect",
         "notebook",
+        "websocket",
         "eventsource"
       ],
       "title": "Backend Type",

software/caddy-frontend/instance-slave-caddy-simplified-input-schema.json

@@ -9,7 +9,7 @@
     },
     "type": {
       "default": "",
-      "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect and notebook, not implemneted is eventsource.",
+      "description": "Type of slave. If redirect, the slave will redirect to the given url. If zope, the rewrite rules will be compatible with Virtual Host Monster. Implemented are default, zope, redirect, notebook and websocket, not implemneted is eventsource.",
       "enum": [
         "",
         "zope"

software/caddy-frontend/templates/apache-custom-slave-list.cfg.in

@@ -248,13 +248,16 @@ rendered = {{ caddy_configuration_directory }}/${:filename}
 
 {%   if caddy_custom_http or caddy_custom_https %}
 template = {{ template_custom_slave_configuration }}
+extra-context =
+    section slave_parameter {{ slave_configuration_section_name }}
 {%   else %}
 template = {{ template_default_slave_configuration }}
+extra-context =
+    section slave_parameter {{ slave_configuration_section_name }}
+    import urllib_module urllib
 {%   endif %}
 
 filename = {{ '%s.conf' % slave_reference }}
-extra-context =
-    section slave_parameter {{ slave_configuration_section_name }}
 {{ '\n' }}
 
 

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -23,8 +23,18 @@
 {%-   do https_host_list.append('https://%s:%s' % (host, slave_parameter['https_port'] )) %}
 {%- endfor %} {#- for host in host_list #}
 {%- set default_path = slave_parameter.get('default-path', '').strip('/') | urlencode %}
-{%- if slave_type == 'notebook' %}
-{# notebook needs http 1.1 max #}
+{%- set websocket_path_list = [] %}
+{%- for websocket_path in slave_parameter.get('websocket-path-list', '').split() %}
+{%-   set websocket_path = websocket_path.strip('/') %}
+{#- Unquote the path, so %20 and similar can be represented correctly #}
+{%-   set websocket_path = urllib_module.unquote(websocket_path.strip()) %}
+{%-   if websocket_path %}
+{%-     do websocket_path_list.append(websocket_path) %}
+{%-   endif %}
+{%- endfor %}
+{%- set websocket_transparent = slave_parameter.get('websocket-transparent', 'true').lower() in TRUE_VALUES %}
+{%- if slave_type in ['notebook', 'websocket'] %}
+{# websocket style needs http 1.1 max #}
 {%-   set enable_h2 = False %}
 {%- endif %}
 
@@ -72,7 +82,7 @@
     if {>Accept-Encoding} not_match "(^gzip,.*|.*, gzip,.*|.*, gzip$|^gzip$)"
     to {1}
   }
-{%    elif slave_type != 'notebook' %}
+{%    elif slave_type not in ['notebook', 'websocket'] %}
   rewrite {
     regexp (.*)
     to {1}
@@ -182,6 +192,38 @@
     without /proxy/
     insecure_skip_verify
   }
+{%- elif slave_type == 'websocket' %}
+{%-   if websocket_path_list %}
+  proxy / {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+{%-     if websocket_transparent %}
+    transparent
+{%-     endif %}
+    insecure_skip_verify
+  }
+{%-     for websocket_path in websocket_path_list %}
+  proxy /{{ websocket_path }} {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    websocket
+{%-       if websocket_transparent %}
+    transparent
+{%-       endif %}
+    insecure_skip_verify
+  }
+{%-     endfor %}
+{%-   else %}
+  proxy / {{ backend_url }} {
+    try_duration {{ slave_parameter['proxy_try_duration'] }}s
+    try_interval {{ slave_parameter['proxy_try_interval'] }}ms
+    websocket
+{%-   if websocket_transparent %}
+    transparent
+{%-   endif %}
+    insecure_skip_verify
+  }
+{%-   endif %}
 {%- else %} {#- if slave_type ==  'zope' and backend_url #}
   # Default configuration
 {%-   if default_path %}

software/caddy-frontend/test/test.py

@@ -1090,6 +1090,26 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'url': cls.backend_url,
         'type': 'notebook',
       },
+      'type-websocket': {
+        'url': cls.backend_url,
+        'type': 'websocket',
+      },
+      'type-websocket-websocket-path-list': {
+        'url': cls.backend_url,
+        'type': 'websocket',
+        'websocket-path-list': '////ws//// /with%20space/',
+      },
+      'type-websocket-websocket-transparent-false': {
+        'url': cls.backend_url,
+        'type': 'websocket',
+        'websocket-transparent': 'false',
+      },
+      'type-websocket-websocket-path-list-websocket-transparent-false': {
+        'url': cls.backend_url,
+        'type': 'websocket',
+        'websocket-path-list': '////ws//// /with%20space/',
+        'websocket-transparent': 'false',
+      },
       'type-eventsource': {
         'url': cls.backend_url,
         'type': 'eventsource',
@@ -1207,9 +1227,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     expected_parameter_dict = {
       'monitor-base-url': None,
       'domain': 'example.com',
-      'accepted-slave-amount': '44',
+      'accepted-slave-amount': '48',
       'rejected-slave-amount': '4',
-      'slave-amount': '48',
+      'slave-amount': '52',
       'kedifa-caucase-url': 'http://[%s]:%s' % (
         SLAPOS_TEST_IPV6, CAUCASE_PORT),
       'rejected-slave-dict': {
@@ -2053,10 +2073,200 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertFalse(
       isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
 
-  @skip('Feature postponed')
   def test_type_websocket(self):
-    # Pure websocket configurable frontend
-    raise NotImplementedError
+    parameter_dict = self.assertSlaveBase(
+      'type-websocket')
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/test-path'
+    )
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertTrue('x-real-ip' in j['Incoming Headers'])
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+
+  def test_type_websocket_websocket_transparent_false(self):
+    parameter_dict = self.assertSlaveBase(
+      'type-websocket-websocket-transparent-false')
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/test-path'
+    )
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertFalse('x-real-ip' in j['Incoming Headers'])
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+
+  def test_type_websocket_websocket_path_list(self):
+    parameter_dict = self.assertSlaveBase(
+      'type-websocket-websocket-path-list')
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertFalse('connection' in j['Incoming Headers'].keys())
+    self.assertTrue('x-real-ip' in j['Incoming Headers'])
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'ws/test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/ws/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertTrue('x-real-ip' in j['Incoming Headers'])
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'with%20space/test-path', headers={'Connection': 'Upgrade'})
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/with%20space/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertTrue('x-real-ip' in j['Incoming Headers'])
+
+  def test_type_websocket_websocket_path_list_websocket_transparent_false(
+    self):
+    parameter_dict = self.assertSlaveBase(
+      'type-websocket-websocket-path-list-websocket-transparent-false')
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqual(
+      self.certificate_pem,
+      der2pem(result.peercert))
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertFalse('connection' in j['Incoming Headers'].keys())
+    self.assertFalse('x-real-ip' in j['Incoming Headers'])
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'], 'ws/test-path',
+      headers={'Connection': 'Upgrade'})
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/ws/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertFalse('x-real-ip' in j['Incoming Headers'])
+
+    result = self.fakeHTTPSResult(
+      parameter_dict['domain'], parameter_dict['public-ipv4'],
+      'with%20space/test-path', headers={'Connection': 'Upgrade'})
+
+    self.assertEqualResultJson(
+      result,
+      'Path',
+      '/with%20space/test-path'
+    )
+    self.assertFalse(
+      isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
+    try:
+      j = result.json()
+    except Exception:
+      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
+    self.assertEqual(
+      'Upgrade',
+      j['Incoming Headers']['connection']
+    )
+    self.assertFalse('x-real-ip' in j['Incoming Headers'])
 
   @skip('Feature postponed')
   def test_type_eventsource(self):

software/caddy-frontend/test/test_data/test.TestSlave.test_file_list_log-CADDY.txt

@@ -72,6 +72,14 @@ T-2/var/log/httpd/_type-notebook_access_log
 T-2/var/log/httpd/_type-notebook_error_log
 T-2/var/log/httpd/_type-redirect_access_log
 T-2/var/log/httpd/_type-redirect_error_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_access_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_error_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list_access_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list_error_log
+T-2/var/log/httpd/_type-websocket-websocket-transparent-false_access_log
+T-2/var/log/httpd/_type-websocket-websocket-transparent-false_error_log
+T-2/var/log/httpd/_type-websocket_access_log
+T-2/var/log/httpd/_type-websocket_error_log
 T-2/var/log/httpd/_type-zope-default-path_access_log
 T-2/var/log/httpd/_type-zope-default-path_error_log
 T-2/var/log/httpd/_type-zope-path_access_log

software/caddy-frontend/test/test_data/test.TestSlave.test_promise_run_plugin-CADDY.txt

@@ -80,6 +80,14 @@ T-2/etc/plugin/check-_type-notebook-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-notebook-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-redirect-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-redirect-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-websocket-transparent-false-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-websocket-transparent-false-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-transparent-false-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-transparent-false-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-zope-default-path-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-zope-default-path-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-zope-error-log-last-day.py: OK

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_file_list_log-CADDY.txt

@@ -72,6 +72,14 @@ T-2/var/log/httpd/_type-notebook_access_log
 T-2/var/log/httpd/_type-notebook_error_log
 T-2/var/log/httpd/_type-redirect_access_log
 T-2/var/log/httpd/_type-redirect_error_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_access_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list-websocket-transparent-false_error_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list_access_log
+T-2/var/log/httpd/_type-websocket-websocket-path-list_error_log
+T-2/var/log/httpd/_type-websocket-websocket-transparent-false_access_log
+T-2/var/log/httpd/_type-websocket-websocket-transparent-false_error_log
+T-2/var/log/httpd/_type-websocket_access_log
+T-2/var/log/httpd/_type-websocket_error_log
 T-2/var/log/httpd/_type-zope-default-path_access_log
 T-2/var/log/httpd/_type-zope-default-path_error_log
 T-2/var/log/httpd/_type-zope-path_access_log

software/caddy-frontend/test/test_data/test.TestSlaveGlobalDisableHttp2.test_promise_run_plugin-CADDY.txt

@@ -80,6 +80,14 @@ T-2/etc/plugin/check-_type-notebook-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-notebook-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-redirect-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-redirect-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-websocket-transparent-false-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-path-list-websocket-transparent-false-error-log-last-hour.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-transparent-false-error-log-last-day.py: OK
+T-2/etc/plugin/check-_type-websocket-websocket-transparent-false-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-zope-default-path-error-log-last-day.py: OK
 T-2/etc/plugin/check-_type-zope-default-path-error-log-last-hour.py: OK
 T-2/etc/plugin/check-_type-zope-error-log-last-day.py: OK