caddy-frontend: Keep Server header intact

Server header might be changed by client facing caddy, for which caddy code
has been fixed to not transmit the Server header at all by default.
In ATS Server header has been disabled by configuration.
haproxy does not add Server header at all.
Thanks to improvement in the test backend, it's asserted that Server header
send by the backend is left intact.

component/caddy/buildout.cfg

@@ -10,7 +10,7 @@ parts =
 <= go-git-package
 go.importpath = github.com/caddyserver/caddy
 repository    = https://lab.nexedi.com/nexedi/caddy.git
-revision      = nxd-v1.0.3-1-g2c11cedc
+revision      = nxd-v1.0.3-1-03fba31bf
 
 [gowork]
 golang  = ${golang1.17:location}

software/caddy-frontend/buildout.hash.cfg

@@ -62,7 +62,7 @@ md5sum = 975177dedf677d24e14cede5d13187ce
 
 [template-trafficserver-records-config]
 _update_hash_filename_ = templates/trafficserver/records.config.jinja2
-md5sum = 88a2db868720009f6092843784b06611
+md5sum = e87238c53d080ef9ef90040e57bc1395
 
 [template-trafficserver-storage-config]
 _update_hash_filename_ = templates/trafficserver/storage.config.jinja2

software/caddy-frontend/templates/trafficserver/records.config.jinja2

@@ -16,6 +16,8 @@ CONFIG proxy.config.output.logfile STRING traffic.out
 CONFIG proxy.config.admin.user_id STRING {{ '#%s' % os_module.geteuid() }}
 LOCAL proxy.local.incoming_ip_to_bind STRING {{ ats_configuration['local-ip'] }}
 CONFIG proxy.config.log.logfile_dir STRING {{ ats_directory['log'] }}
+# Never change Server header
+CONFIG proxy.config.http.response_server_enabled INT 0
 # Implement RFC 5861 with core
 CONFIG proxy.config.http.cache.open_write_fail_action INT 2
 CONFIG proxy.config.body_factory.template_sets_dir STRING  {{ ats_configuration['templates-dir'] }}

software/caddy-frontend/test/test.py

@@ -485,6 +485,9 @@ def fakeHTTPResult(domain, path, port=HTTP_PORT,
 class TestHandler(BaseHTTPRequestHandler):
   identification = None
   configuration = {}
+  # override Server header response
+  server_version = "TestBackend"
+  sys_version = ""
 
   def log_message(self, *args):
     if os.environ.get('SLAPOS_TEST_DEBUG'):
@@ -843,6 +846,19 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     except Exception as e:
       self.fail(e)
 
+  def assertResponseHeaders(self, result):
+    headers = result.headers.copy()
+    self.assertKeyWithPop('Date', headers)
+    # drop vary-keys
+    headers.pop('Connection', None)
+    headers.pop('Content-Length', None)
+    headers.pop('Keep-Alive', None)
+    headers.pop('Transfer-Encoding', None)
+
+    self.assertEqual('TestBackend', headers.pop('Server', ''))
+
+    return headers
+
   def assertLogAccessUrlWithPop(self, parameter_dict):
     log_access_url = parameter_dict.pop('log-access-url')
 
@@ -3612,18 +3628,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
+    headers = self.assertResponseHeaders(result)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -3654,18 +3661,9 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
-
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -3712,18 +3710,8 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
+    headers = self.assertResponseHeaders(result)
     self.assertKeyWithPop('Age', headers)
-
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -3743,7 +3731,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     self.assertEqual(httplib.OK, result.status_code)
     self.assertEqualResultJson(result, 'Path', '/HTTPS/test')
 
-    headers = result.headers.copy()
+    self.assertResponseHeaders(result)
 
     result = fakeHTTPSResult(
       parameter_dict['domain'],
@@ -3753,8 +3741,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqual(httplib.OK, result.status_code)
     self.assertEqualResultJson(result, 'Path', '/HTTP/test')
-
-    headers = result.headers.copy()
+    self.assertResponseHeaders(result)
 
   def test_enable_cache(self):
     parameter_dict = self.assertSlaveBase('enable_cache')
@@ -3771,18 +3758,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -3973,18 +3952,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path/deeper')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -4118,18 +4089,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -4165,18 +4128,10 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
+    headers = self.assertResponseHeaders(result)
 
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
     self.assertKeyWithPop('Age', headers)
 
-    # drop keys appearing randomly in headers
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Content-Length', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -4206,16 +4161,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
-
-    # drop vary-keys
-    headers.pop('Content-Length', None)
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
+    headers = self.assertResponseHeaders(result)
 
     self.assertEqual(
       {
@@ -4240,17 +4186,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
-
-    # drop vary-keys
-    headers.pop('Content-Length', None)
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
+    headers = self.assertResponseHeaders(result)
     self.assertEqual(
       {
         'Content-type': 'application/json',
@@ -5017,17 +4953,7 @@ class TestSlaveGlobalDisableHttp2(TestSlave):
 
     self.assertEqualResultJson(result, 'Path', '/test-path')
 
-    headers = result.headers.copy()
-
-    self.assertKeyWithPop('Server', headers)
-    self.assertKeyWithPop('Date', headers)
-
-    # drop vary-keys
-    headers.pop('Content-Length', None)
-    headers.pop('Transfer-Encoding', None)
-    headers.pop('Connection', None)
-    headers.pop('Keep-Alive', None)
-
+    headers = self.assertResponseHeaders(result)
     self.assertEqual(
       {
         'Content-type': 'application/json',