caddy-frontend: Set https-only by default to true

As https-only becomes default, tests are adapted in order to follow new
approach, case by case, or the redirect http->https is asserted, or https
access is used instead of http.

software/caddy-frontend/buildout.hash.cfg

@@ -54,7 +54,7 @@ md5sum = f20d6c3d2d94fb685f8d26dfca1e822b
 
 [template-default-slave-virtualhost]
 filename = templates/default-virtualhost.conf.in
-md5sum = b5447e33658b8a81b75275630f9da119
+md5sum = 7e26935bb6daf00d8fc01d97eebc7abd
 
 [template-cached-slave-virtualhost]
 filename = templates/cached-virtualhost.conf.in

software/caddy-frontend/instance-slave-caddy-input-schema.json

@@ -44,7 +44,7 @@
       "type": "string"
     },
     "https-only": {
-      "default": "false",
+      "default": "true",
       "description": "If set to true, http requests will be redirected to https",
       "enum": [
         "false",

software/caddy-frontend/instance-slave-caddy-simplified-input-schema.json

@@ -72,7 +72,7 @@
       "type": "string"
     },
     "https-only": {
-      "default": "false",
+      "default": "true",
       "description": "If set to true, http requests will be redirected to https",
       "enum": [
         "false",

software/caddy-frontend/templates/default-virtualhost.conf.in

@@ -10,7 +10,7 @@
 {%- set enable_h2 = slave_parameter['global_disable_http2'].lower() not in TRUE_VALUES and slave_parameter.get('enable-http2', slave_parameter['enable_http2_by_default']).lower() in TRUE_VALUES %}
 {%- set ssl_proxy_verify = slave_parameter.get('ssl-proxy-verify', '').lower() in TRUE_VALUES %}
 {%- set disabled_cookie_list =  slave_parameter.get('disabled-cookie-list', '').split() %}
-{%- set https_only = slave_parameter.get('https-only', '').lower() in TRUE_VALUES %}
+{%- set https_only = slave_parameter.get('https-only', 'true').lower() in TRUE_VALUES %}
 {%- set slave_type = slave_parameter.get('type', '') %}
 {%- set host_list = server_alias_list %}
 {%- set cipher_list = slave_parameter.get('cipher_list', '').strip() %}

software/caddy-frontend/test/test.py

@@ -1163,7 +1163,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       },
       'https-only': {
         'url': cls.backend_url,
-        'https-only': True,
+        'https-only': False,
       },
       'custom_domain': {
         'url': cls.backend_url,
@@ -1208,7 +1208,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
         'url': cls.backend_url,
         'prefer-gzip-encoding-to-backend': 'true',
         'type': 'zope',
-        'https-only': 'true',
+        'https-only': 'false',
       },
       'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt': {
         'url': cls.backend_https_url,
@@ -1230,7 +1230,8 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'type-zope-virtualhostroot-http-port': {
         'url': cls.backend_url,
         'type': 'zope',
-        'virtualhostroot-http-port': '12345'
+        'virtualhostroot-http-port': '12345',
+        'https-only': 'false',
       },
       'type-zope-virtualhostroot-https-port': {
         'url': cls.backend_url,
@@ -1364,7 +1365,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'prefer-gzip-encoding-to-backend-https-only': {
         'url': cls.backend_url,
         'prefer-gzip-encoding-to-backend': 'true',
-        'https-only': 'true',
+        'https-only': 'false',
       },
       'disabled-cookie-list': {
         'url': cls.backend_url,
@@ -1624,7 +1625,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       log_regexp)
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
+    self.assertEqual(
+      httplib.FOUND,
+      result_http.status_code
+    )
+
+    self.assertEqual(
+      'https://empty.example.com/test-path',
+      result_http.headers['Location']
+    )
 
     # check that 404 is as configured
     result_missing = fakeHTTPSResult(
@@ -1693,19 +1702,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper')
-    self.assertEqualResultJson(result_http, 'Path', '/test-path/deeper')
 
-    try:
-      j = result_http.json()
-    except Exception:
-      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
-    self.assertFalse('remote_user' in j['Incoming Headers'].keys())
-
-    self.assertFalse('Content-Encoding' in result_http.headers)
+    self.assertEqual(
+      httplib.FOUND,
+      result_http.status_code
+    )
 
     self.assertEqual(
-      'secured=value;secure, nonsecured=value',
-      result_http.headers['Set-Cookie']
+      'https://url.example.com/test-path/deeper',
+      result_http.headers['Location']
     )
 
     # check that try_duration == 5 in the test_url slave
@@ -2180,15 +2185,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper')
 
-    self.assertEqual(
-      httplib.FOUND,
-      result_http.status_code
-    )
-
-    self.assertEqual(
-      'https://httpsonly.example.com/test-path/deeper',
-      result_http.headers['Location']
-    )
+    self.assertEqualResultJson(result_http, 'Path', '/test-path/deeper')
 
   def test_custom_domain(self):
     reference = 'custom_domain'
@@ -2350,16 +2347,19 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper')
 
-    self.assertEqualResultJson(
-      result,
-      'Path',
-      '/VirtualHostBase/http//typezope.example.com:80/'
-      '/VirtualHostRoot/test-path/deeper'
+    self.assertEqual(
+      httplib.FOUND,
+      result.status_code
     )
 
-  def test_type_zope_prefer_gzip_encoding_to_backend(self):
+    self.assertEqual(
+      'https://typezope.example.com/test-path/deep/.././deeper',
+      result.headers['Location']
+    )
+
+  def test_type_zope_prefer_gzip_encoding_to_backend_https_only(self):
     parameter_dict = self.assertSlaveBase(
-      'type-zope-prefer-gzip-encoding-to-backend')
+      'type-zope-prefer-gzip-encoding-to-backend-https-only')
 
     result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
@@ -2379,7 +2379,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       result,
       'Path',
       '/VirtualHostBase/https//'
-      'typezopeprefergzipencodingtobackend.example.com:443/'
+      'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
       '/VirtualHostRoot/test-path/deeper'
     )
 
@@ -2391,7 +2391,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       result,
       'Path',
       '/VirtualHostBase/http//'
-      'typezopeprefergzipencodingtobackend.example.com:80/'
+      'typezopeprefergzipencodingtobackendhttpsonly.example.com:80/'
       '/VirtualHostRoot/test-path/deeper'
     )
 
@@ -2414,7 +2414,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       result,
       'Path',
       '/VirtualHostBase/https//'
-      'typezopeprefergzipencodingtobackend.example.com:443/'
+      'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
       '/VirtualHostRoot/test-path/deeper'
     )
     self.assertEqual(
@@ -2429,15 +2429,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       result,
       'Path',
       '/VirtualHostBase/http//'
-      'typezopeprefergzipencodingtobackend.example.com:80/'
+      'typezopeprefergzipencodingtobackendhttpsonly.example.com:80/'
       '/VirtualHostRoot/test-path/deeper'
     )
     self.assertEqual(
       'gzip', result.json()['Incoming Headers']['accept-encoding'])
 
-  def test_type_zope_prefer_gzip_encoding_to_backend_https_only(self):
+  def test_type_zope_prefer_gzip_encoding_to_backend(self):
     parameter_dict = self.assertSlaveBase(
-      'type-zope-prefer-gzip-encoding-to-backend-https-only')
+      'type-zope-prefer-gzip-encoding-to-backend')
 
     result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
@@ -2457,7 +2457,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       result,
       'Path',
       '/VirtualHostBase/https//'
-      'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
+      'typezopeprefergzipencodingtobackend.example.com:443/'
       '/VirtualHostRoot/test-path/deeper'
     )
 
@@ -2494,7 +2494,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       result,
       'Path',
       '/VirtualHostBase/https//'
-      'typezopeprefergzipencodingtobackendhttpsonly.example.com:443/'
+      'typezopeprefergzipencodingtobackend.example.com:443/'
       '/VirtualHostRoot/test-path/deeper'
     )
     self.assertEqual(
@@ -2889,10 +2889,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      httplib.BAD_GATEWAY,
+      httplib.FOUND,
       result_http.status_code
     )
 
+    self.assertEqual(
+      'https://sslproxyverifysslproxycacrtunverified.example.com/test-path',
+      result_http.headers['Location']
+    )
+
   def test_ssl_proxy_verify_ssl_proxy_ca_crt(self):
     parameter_dict = self.assertSlaveBase('ssl-proxy-verify_ssl_proxy_ca_crt')
 
@@ -2920,19 +2925,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqualResultJson(result_http, 'Path', '/test-path')
-
-    try:
-      j = result_http.json()
-    except Exception:
-      raise ValueError('JSON decode problem in:\n%s' % (result.text,))
-    self.assertFalse('remote_user' in j['Incoming Headers'].keys())
 
-    self.assertFalse('Content-Encoding' in result.headers)
+    self.assertEqual(
+      httplib.FOUND,
+      result_http.status_code
+    )
 
     self.assertEqual(
-      'secured=value;secure, nonsecured=value',
-      result_http.headers['Set-Cookie']
+      'https://sslproxyverifysslproxycacrt.example.com/test-path',
+      result_http.headers['Location']
     )
 
   @skip('Not implemented in new test system')
@@ -2999,10 +3000,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'test-path/deeper')
 
     self.assertEqual(
-      httplib.BAD_GATEWAY,
+      httplib.FOUND,
       result_http.status_code
     )
 
+    self.assertEqual(
+      'https://enablecachesslproxyverifysslproxycacrtunverified.example.com/'
+      'test-path/deeper',
+      result_http.headers['Location']
+    )
+
   def test_enable_cache_ssl_proxy_verify_ssl_proxy_ca_crt(self):
     parameter_dict = self.assertSlaveBase(
       'enable_cache-ssl-proxy-verify_ssl_proxy_ca_crt')
@@ -3131,10 +3138,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
     self.assertEqual(
-      httplib.BAD_GATEWAY,
+      httplib.FOUND,
       result_http.status_code
     )
 
+    self.assertEqual(
+      'https://typezopesslproxyverifysslproxycacrtunverified.example.com/'
+      'test-path',
+      result_http.headers['Location']
+    )
+
   def test_type_zope_ssl_proxy_verify_ssl_proxy_ca_crt(self):
     parameter_dict = self.assertSlaveBase(
       'type-zope-ssl-proxy-verify_ssl_proxy_ca_crt')
@@ -3163,12 +3176,14 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     result = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
 
-    self.assertEqualResultJson(
-      result,
-      'Path',
-      '/VirtualHostBase/http//'
-      'typezopesslproxyverifysslproxycacrt.example.com:80/'
-      '/VirtualHostRoot/test-path'
+    self.assertEqual(
+      httplib.FOUND,
+      result.status_code
+    )
+
+    self.assertEqual(
+      'https://typezopesslproxyverifysslproxycacrt.example.com/test-path',
+      result.headers['Location']
     )
 
   def test_type_zope_ssl_proxy_verify_unverified(self):
@@ -3201,7 +3216,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
+    self.assertEqual(
+      httplib.FOUND,
+      result_http.status_code
+    )
+
+    self.assertEqual(
+      'https://monitoripv6test.example.com/test-path',
+      result_http.headers['Location']
+    )
 
     monitor_file = glob.glob(
       os.path.join(
@@ -3230,7 +3253,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
+    self.assertEqual(
+      httplib.FOUND,
+      result_http.status_code
+    )
+
+    self.assertEqual(
+      'https://monitoripv4test.example.com/test-path',
+      result_http.headers['Location']
+    )
 
     monitor_file = glob.glob(
       os.path.join(
@@ -3260,7 +3291,15 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
+    self.assertEqual(
+      httplib.FOUND,
+      result_http.status_code
+    )
+
+    self.assertEqual(
+      'https://re6stoptimaltest.example.com/test-path',
+      result_http.headers['Location']
+    )
 
     monitor_file = glob.glob(
       os.path.join(
@@ -3290,7 +3329,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(httplib.NOT_FOUND, result_http.status_code)
+
+    self.assertEqual(
+      httplib.FOUND,
+      result_http.status_code
+    )
+
+    self.assertEqual(
+      'https://ciphers.example.com/test-path',
+      result_http.headers['Location']
+    )
 
     configuration_file = glob.glob(
       os.path.join(
@@ -3319,7 +3367,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       parameter_dict
     )
 
-    result = fakeHTTPResult(
+    result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper', headers={
         'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
@@ -3360,7 +3408,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
   def test_enable_cache_server_alias(self):
     parameter_dict = self.assertSlaveBase('enable_cache_server_alias')
 
-    result = fakeHTTPResult(
+    result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper', headers={
         'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
@@ -3403,43 +3451,20 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
       'test-path/deep/.././deeper', headers={
         'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
         'revalidate=3600, stale-if-error=3600'})
-
-    self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
-
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
-    self.assertKeyWithPop('Age', headers)
-
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
-      {
-        'Content-type': 'application/json',
-        'Set-Cookie': 'secured=value;secure, nonsecured=value',
-        'Cache-Control': 'max-age=1, stale-while-revalidate=3600, '
-                         'stale-if-error=3600'
-      },
-      headers
+      httplib.FOUND,
+      result.status_code
     )
 
-    backend_headers = result.json()['Incoming Headers']
-    via = backend_headers.pop('via', None)
-    self.assertNotEqual(via, None)
-    self.assertRegexpMatches(
-      via,
-      r'^http\/1.1 caddy-frontend-1\[.*\] \(ApacheTrafficServer\/7.1.6\)$'
+    self.assertEqual(
+      'https://enablecacheserveralias1.example.com/test-path/deeper',
+      result.headers['Location']
     )
 
   def test_enable_cache(self):
     parameter_dict = self.assertSlaveBase('enable_cache')
 
-    result = fakeHTTPResult(
+    result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper', headers={
         'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
@@ -3499,8 +3524,12 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
             'revalidate=3600, stale-if-error=3600'})
 
       result = fakeHTTPResult(
-        parameter_dict['domain'], parameter_dict['public-ipv4'],
-        'test-path/deeper',  # simple path, as ATS can't change them
+        # append with :HTTP_PORT to mimic access in ATS
+        parameter_dict['domain'] + ':' + HTTPS_PORT,
+        parameter_dict['public-ipv4'],
+        # prepend with HTTPS to mimic access via https in ATS
+        # use simple path, as it is changed in Caddy
+        'HTTPS/test-path/deeper',
         port=23432, headers={
           'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
           'revalidate=3600, stale-if-error=3600'})
@@ -3588,7 +3617,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     parameter_dict = self.assertSlaveBase('enable_cache')
     # check that timeout seen by ATS does not result in many queries done
     # to the backend and that next request works like a charm
-    result = fakeHTTPResult(
+    result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test_enable_cache_ats_timeout', headers={
         'Timeout': '15',
@@ -3655,7 +3684,7 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertIn(matching_line_amount, [0, 1])
 
     # the result is available immediately after
-    result = fakeHTTPResult(
+    result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper', headers={
         'X-Reply-Header-Cache-Control': 'max-age=1, stale-while-'
@@ -3819,8 +3848,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     self.assertTrue(
       isHTTP2(parameter_dict['domain'], parameter_dict['public-ipv4']))
 
-  def test_prefer_gzip_encoding_to_backend(self):
-    parameter_dict = self.assertSlaveBase('prefer-gzip-encoding-to-backend')
+  def test_prefer_gzip_encoding_to_backend_https_only(self):
+    parameter_dict = self.assertSlaveBase(
+      'prefer-gzip-encoding-to-backend-https-only')
 
     result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
@@ -3894,9 +3924,9 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-  def test_prefer_gzip_encoding_to_backend_https_only(self):
+  def test_prefer_gzip_encoding_to_backend(self):
     parameter_dict = self.assertSlaveBase(
-      'prefer-gzip-encoding-to-backend-https-only')
+      'prefer-gzip-encoding-to-backend')
 
     result = fakeHTTPSResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
@@ -4212,7 +4242,16 @@ http://apachecustomhttpsaccepted.example.com:%%(http_port)s {
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'],
       'test-path/deep/.././deeper')
-    self.assertEqualResultJson(result_http, 'Path', '/http/test-path/deeper')
+
+    self.assertEqual(
+      httplib.FOUND,
+      result_http.status_code
+    )
+
+    self.assertEqual(
+      'https://urlhttpsurl.example.com/test-path/deeper',
+      result_http.headers['Location']
+    )
 
 
 @skip('Impossible to instantiate cluster with stopped partition')
@@ -6601,7 +6640,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
 
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(httplib.OK, result_http.status_code)
+    self.assertEqual(httplib.FOUND, result_http.status_code)
 
     configuration_file = glob.glob(
       os.path.join(
@@ -6627,7 +6666,7 @@ class TestSlaveCiphers(SlaveHttpFrontendTestCase, TestDataMixin):
 
     result_http = fakeHTTPResult(
       parameter_dict['domain'], parameter_dict['public-ipv4'], 'test-path')
-    self.assertEqual(httplib.OK, result_http.status_code)
+    self.assertEqual(httplib.FOUND, result_http.status_code)
 
     configuration_file = glob.glob(
       os.path.join(