software/rapid-cdn: Fix alpn, especially for QUIC/HTTP3

There was wrong configuration of alpn h3, which defined globally, but
overridden by slave specific configuration. Thanks to this fix, on a node
which is QUIC enabled, the QUIC possible slaves are working well.

software/rapid-cdn/buildout.hash.cfg

@@ -22,7 +22,7 @@ md5sum = 5784bea3bd608913769ff9a8afcccb68
 
 [profile-frontend]
 filename = instance-frontend.cfg.in
-md5sum = 08d4476548f21a85a2e4701e5f65154d
+md5sum = 2f0b4af26c5e947f77cf85ab44e0fe5d
 
 [profile-master]
 filename = instance-master.cfg.in
@@ -30,7 +30,7 @@ md5sum = 2aaab85bad51136b38f6a16d662a7b3e
 
 [profile-slave-list]
 filename = instance-slave-list.cfg.in
-md5sum = ca2e775e7bd2a96e46113a628461a46f
+md5sum = b26f4536102ff2cdc1356f6626928975
 
 [profile-master-publish-slave-information]
 filename = instance-master-publish-slave-information.cfg.in
@@ -38,11 +38,11 @@ md5sum = cba4d995962f7fbeae3f61c9372c4181
 
 [template-frontend-haproxy-configuration]
 _update_hash_filename_ = templates/frontend-haproxy.cfg.in
-md5sum = 6f9205190e42841d6e368b8eda710b73
+md5sum = 2e964dbe75f725c4e45e62720a77bba3
 
 [template-frontend-haproxy-crt-list]
 _update_hash_filename_ = templates/frontend-haproxy-crt-list.in
-md5sum = 13c294af9950939c76021eb19305f3ab
+md5sum = 238760d48d2875f087ad2d784e2a8fcd
 
 [template-not-found-html]
 _update_hash_filename_ = templates/notfound.html

software/rapid-cdn/instance-frontend.cfg.in

@@ -372,6 +372,7 @@ organizational-unit = {{ instance_parameter_dict['configuration.frontend-name']
 backend-client-caucase-url = {{ slapparameter_dict['backend-client-caucase-url'] }}
 partition_ipv6 =  ${slap-configuration:ipv6-random}
 url-ready-file = ${directory:var}/url-ready.txt
+quic = {{ FRONTEND_HAPROXY_QUIC }}
 extra-context =
     key backend_client_caucase_url :backend-client-caucase-url
     import furl_module furl
@@ -385,6 +386,7 @@ extra-context =
     key empty_template software-release-path:template-empty
     key template_expose_csr_nginx_conf software-release-path:template-expose-csr-nginx-conf
     key software_type :software_type
+    key quic :quic
     key frontend_lazy_graceful_reload frontend-haproxy-lazy-graceful:output
     key monitor_base_url monitor-instance-parameter:monitor-base-url
     key node_id frontend-node-id:value

software/rapid-cdn/instance-slave-list.cfg.in

@@ -9,6 +9,12 @@
 {%- set backend_haproxy_http_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['http-port']) %}
 {%- set backend_haproxy_https_url = 'http://%s:%s' % (instance_parameter_dict['ipv4-random'], backend_haproxy_configuration['https-port']) %}
 {%- set TRUE_VALUES = ['y', 'yes', '1', 'true'] %}
+{%- set ALPN_HTTP11 = "alpn http/1.1,http/1.0" %}
+{%- if quic.lower() in TRUE_VALUES %}
+{%-   set ALPN_HTTP2PLUS = "alpn h3,h2,http/1.1,http/1.0" %}
+{%- else %}
+{%-   set ALPN_HTTP2PLUS = "alpn h2,http/1.1,http/1.0" %}
+{%- endif %}
 {%- set generic_instance_parameter_dict = { 'cache_access': cache_access, 'local_ipv4': instance_parameter_dict['ipv4-random'], 'http_port': configuration['plain_http_port'], 'https_port': configuration['port']} %}
 {%- set slave_log_dict = {} %}
 {%- set slave_instance_information_list = [] %}
@@ -243,10 +249,12 @@ context =
 {%-     endif %}
 {%-   endfor %}
 {%-   do slave_instance.__setitem__('websocket-path-list', websocket_path_list) %}
-{%-   do slave_instance.__setitem__('enable_h2', slave_instance['enable-http2']) %}
-{%-   if slave_instance['type'] in ['notebook', 'websocket'] %}
-{#    websocket style needs http 1.1 max #}
-{%-     do slave_instance.__setitem__('enable_h2', False) %}
+{#- Handle alpn negotiation -#}
+{%-   if slave_instance['type'] in ['notebook', 'websocket'] or not slave_instance['enable-http2'] %}
+{#    websocket style needs http 1.1 max, just like non-http2 frontends #}
+{%-     do slave_instance.__setitem__('alpn', ALPN_HTTP11) %}
+{%-   else %}
+{%-     do slave_instance.__setitem__('alpn', ALPN_HTTP2PLUS) %}
 {%-   endif %}
 
 [slave-log-directory-dict]

software/rapid-cdn/software.cfg

@@ -108,6 +108,7 @@ haproxy_executable = ${haproxy:location}/sbin/haproxy
 haproxy_quic_executable = ${haproxy-quic:location}/sbin/haproxy
 rsyslogd_executable = ${rsyslogd:location}/sbin/rsyslogd
 curl = ${curl:location}
+curl_http3 = ${curl-http3:location}
 dash = ${dash:location}
 gzip = ${gzip:location}
 logrotate = ${logrotate:location}

software/rapid-cdn/templates/frontend-haproxy-crt-list.in

@@ -6,11 +6,7 @@
 {%-   if slave['ciphers'] %}
 {%-     do sslbindconf.append('ciphers %s' % (slave['ciphers']),) %}
 {%-   endif %}
-{%-   if slave['enable_h2'] %}
-{%-     do sslbindconf.append('alpn h2,http/1.1,http/1.0') %}
-{%-   else %}
-{%-     do sslbindconf.append('alpn http/1.1,http/1.0') %}
-{%-   endif %}
+{%-   do sslbindconf.append(slave['alpn']) %}
 {%-   do entry_list.append('[' + ' '.join(sslbindconf) + ']') %}
 {#-   <snifilter> #}
 {%-   do entry_list.extend(slave['host_list']) %}

software/rapid-cdn/templates/frontend-haproxy.cfg.in

@@ -82,7 +82,7 @@ frontend https-frontend
 {%- if QUIC %}
   bind quic4@{{ configuration['local-ipv4'] }}:{{ configuration['https-port'] }} ssl crt-list {{ crt_list }} alpn h3
   bind quic6@{{ configuration['global-ipv6'] }}:{{ configuration['https-port'] }} ssl crt-list {{ crt_list }} alpn h3
-  http-response set-header alt-svc "h3=\":{{ configuration['quic-port'] }}\"; ma=900"
+  http-response set-header alt-svc "h3=\":{{ configuration['quic-port'] }}\"; ma=3600"
   {#- Ask Chromium to use QUIC #}
   http-response set-header alternate-protocol {{ configuration['quic-port'] }}:quic
 {%- endif %}

software/rapid-cdn/test/test.py

@@ -7367,6 +7367,86 @@ backend _health-check-default-http
     self.assertEqual(result.status_code, http.client.SERVICE_UNAVAILABLE)
 
 
+class TestSlaveQuic(SlaveHttpFrontendTestCase, TestDataMixin, AtsMixin):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'domain': 'example.com',
+      'port': HTTPS_PORT,
+      'plain_http_port': HTTP_PORT,
+      'kedifa_port': KEDIFA_PORT,
+      'caucase_port': CAUCASE_PORT,
+      'request-timeout': '12',
+      '-frontend-1-experimental-haproxy-quic': True,
+      '-frontend-1-experimental-haproxy-flavour': 'quic',
+      '-frontend-1-experimental-quic-port': HTTPS_PORT,
+    }
+
+  @classmethod
+  def getSlaveParameterDictDict(cls):
+    return {
+      'url': {
+        'url': cls.backend_url,
+      },
+      'enable_cache': {
+        'url': cls.backend_url,
+        'enable_cache': True,
+      },
+    }
+
+  def get_curl_http3(self):
+    # Very hacky way to fetch curl from own software release instead of
+    # polluting slapos-sr-testing
+    with open(os.path.join(self.software_path, '.installed.cfg')) as fh:
+      for line in fh.readlines():
+        if line.startswith('location =') and 'curl-http3' in line:
+          return '/'.join([line.strip().split()[-1], 'bin/curl'])
+
+  def assertHttp3(self, domain, direct=True):
+    alt_svc = tempfile.NamedTemporaryFile(delete=False)
+    curl_command = [self.get_curl_http3()]
+    if direct:
+      curl_command.append('--http3')
+    else:
+      curl_command.extend(['--alt-svc', alt_svc.name])
+    curl_command.extend([
+      '-k',
+      '-v',
+      '-D', '-',
+      '-o', '/dev/null',
+      '-H', 'Host: %s' % (domain,),
+      '--resolve', '%(domain)s:%(https_port)s:%(ip)s' % dict(
+        ip=TEST_IP, domain=domain, https_port=HTTPS_PORT),
+      'https://%(domain)s:%(https_port)s/' % dict(
+        domain=domain, https_port=HTTPS_PORT),
+    ])
+
+    def call_curl():
+      prc = subprocess.Popen(
+        curl_command, stdout=subprocess.PIPE, stderr=subprocess.PIPE
+      )
+      out, err = prc.communicate()
+      assert prc.returncode == 0, "Problem running %r. "\
+          "Output:\n%s\nError:\n%s" % (
+            ' '.join(curl_command), out, err)
+      return [q.strip() for q in out.decode().splitlines()]
+
+    if not direct:
+      # curl with alt-svc does not switch to HTTP3 in one request
+      self.assertEqual('HTTP/2 200', call_curl()[0])
+    self.assertEqual('HTTP/3 200', call_curl()[0])
+
+  def test_url(self):
+    parameter_dict = self.assertSlaveBase('url')
+    self.assertHttp3(parameter_dict['domain'])
+    self.assertHttp3(parameter_dict['domain'], direct=False)
+
+  def test_enable_cache(self):
+    parameter_dict = self.assertSlaveBase('enable_cache')
+    self.assertHttp3(parameter_dict['domain'])
+    self.assertHttp3(parameter_dict['domain'], direct=False)
+
+
 if __name__ == '__main__':
   class HTTP6Server(ThreadedHTTPServer):
     address_family = socket.AF_INET6

software/rapid-cdn/test/test_data/test.TestSlaveQuic.test00cluster_request_instance_parameter_dict.txt

+[
+  {
+    "-frontend-1-experimental-haproxy-flavour": "quic",
+    "-frontend-1-experimental-haproxy-quic": "True",
+    "-frontend-1-experimental-quic-port": "11443",
+    "caucase_port": "15090",
+    "domain": "example.com",
+    "full_address_list": [],
+    "instance_title": "testing partition 0",
+    "ip_list": [
+      [
+        "T-0",
+        "@@_ipv4_address@@"
+      ],
+      [
+        "T-0",
+        "@@_ipv6_address@@"
+      ]
+    ],
+    "kedifa_port": "15080",
+    "plain_http_port": "11080",
+    "port": "11443",
+    "request-timeout": "12",
+    "root_instance_title": "testing partition 0",
+    "slap_computer_id": "local",
+    "slap_computer_partition_id": "T-0",
+    "slap_software_release_url": "@@00getSoftwareURL@@",
+    "slap_software_type": "RootSoftwareInstance",
+    "slave_instance_list": [
+      {
+        "slap_software_type": "RootSoftwareInstance",
+        "slave_reference": "_url",
+        "slave_title": "_url",
+        "url": "http://@@_ipv4_address@@:@@_server_http_port@@/"
+      },
+      {
+        "enable_cache": true,
+        "slap_software_type": "RootSoftwareInstance",
+        "slave_reference": "_enable_cache",
+        "slave_title": "_enable_cache",
+        "url": "http://@@_ipv4_address@@:@@_server_http_port@@/"
+      }
+    ],
+    "timestamp": "@@TIMESTAMP@@"
+  },
+  {
+    "_": {
+      "caucase_port": "15090",
+      "cluster-identification": "testing partition 0",
+      "kedifa_port": "15080",
+      "monitor-cors-domains": "monitor.app.officejs.com",
+      "monitor-httpd-port": "8402",
+      "monitor-password": "@@monitor-password@@",
+      "monitor-username": "admin",
+      "slave-list": [
+        {
+          "enable_cache": true,
+          "slave_reference": "_enable_cache",
+          "url": "http://@@_ipv4_address@@:@@_server_http_port@@/"
+        },
+        {
+          "slave_reference": "_url",
+          "url": "http://@@_ipv4_address@@:@@_server_http_port@@/"
+        }
+      ]
+    },
+    "full_address_list": [],
+    "instance_title": "kedifa",
+    "ip_list": [
+      [
+        "T-1",
+        "@@_ipv4_address@@"
+      ],
+      [
+        "T-1",
+        "@@_ipv6_address@@"
+      ]
+    ],
+    "root_instance_title": "testing partition 0",
+    "slap_computer_id": "local",
+    "slap_computer_partition_id": "T-1",
+    "slap_software_release_url": "@@00getSoftwareURL@@",
+    "slap_software_type": "kedifa",
+    "slave_instance_list": [],
+    "timestamp": "@@TIMESTAMP@@"
+  },
+  {
+    "_": {
+      "backend-client-caucase-url": "http://[@@_ipv6_address@@]:8990",
+      "cluster-identification": "testing partition 0",
+      "domain": "example.com",
+      "extra_slave_instance_list": "[{\"enable_cache\": true, \"slave_reference\": \"_enable_cache\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}, {\"slave_reference\": \"_url\", \"url\": \"http://@@_ipv4_address@@:@@_server_http_port@@/\"}]",
+      "frontend-haproxy-flavour": "quic",
+      "frontend-haproxy-quic": "True",
+      "frontend-name": "caddy-frontend-1",
+      "frontend-quic-port": "11443",
+      "kedifa-caucase-url": "http://[@@_ipv6_address@@]:15090",
+      "master-key-download-url": "https://[@@_ipv6_address@@]:15080/@@master-key-download-url_endpoint@@",
+      "monitor-cors-domains": "monitor.app.officejs.com",
+      "monitor-httpd-port": 8411,
+      "monitor-password": "@@monitor-password@@",
+      "monitor-username": "admin",
+      "plain_http_port": "11080",
+      "port": "11443",
+      "request-timeout": "12",
+      "slave-kedifa-information": "{\"_enable_cache\": {\"kedifa-caucase-url\": \"http://[@@_ipv6_address@@]:15090\", \"key-download-url\": \"https://[@@_ipv6_address@@]:15080/@@enable_cache_key-generate-auth-url@@\", \"key-generate-auth-url\": \"https://[@@_ipv6_address@@]:15080/@@enable_cache_key-generate-auth-url@@/@@enable_cache_key-upload-url@@\", \"key-upload-url\": \"https://[@@_ipv6_address@@]:15080/@@enable_cache_key-generate-auth-url@@?auth=\"}, \"_url\": {\"kedifa-caucase-url\": \"http://[@@_ipv6_address@@]:15090\", \"key-download-url\": \"https://[@@_ipv6_address@@]:15080/@@url_key-generate-auth-url@@\", \"key-generate-auth-url\": \"https://[@@_ipv6_address@@]:15080/@@url_key-generate-auth-url@@/@@enable_cache_key-upload-url@@\", \"key-upload-url\": \"https://[@@_ipv6_address@@]:15080/@@url_key-generate-auth-url@@?auth=\"}}"
+    },
+    "full_address_list": [],
+    "instance_title": "caddy-frontend-1",
+    "ip_list": [
+      [
+        "T-2",
+        "@@_ipv4_address@@"
+      ],
+      [
+        "T-2",
+        "@@_ipv6_address@@"
+      ]
+    ],
+    "root_instance_title": "testing partition 0",
+    "slap_computer_id": "local",
+    "slap_computer_partition_id": "T-2",
+    "slap_software_release_url": "@@00getSoftwareURL@@",
+    "slap_software_type": "single-custom-personal",
+    "slave_instance_list": [],
+    "timestamp": "@@TIMESTAMP@@"
+  }
+]

software/rapid-cdn/test/test_data/test.TestSlaveQuic.test00file_list_log.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/expose-csr.log
+T-2/var/log/frontend-haproxy.log
+T-2/var/log/httpd/_enable_cache_access_log
+T-2/var/log/httpd/_enable_cache_backend_log
+T-2/var/log/httpd/_url_access_log
+T-2/var/log/httpd/_url_backend_log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log

software/rapid-cdn/test/test_data/test.TestSlaveQuic.test00file_list_run.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/fhlog.sck
+T-2/var/run/frontend-haproxy-rsyslogd.pid
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/rapid-cdn/test/test_data/test.TestSlaveQuic.test00supervisor_state.txt

+T-0:aibcc-user-caucase-updater-on-watch RUNNING
+T-0:aikc-user-caucase-updater-on-watch RUNNING
+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:master-introspection-server-{hash-master-introspection}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr-{hash-generic}-on-watch RUNNING
+T-2:frontend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:frontend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:frontend-haproxy-safe-graceful EXITED
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED

software/rapid-cdn/test/test_data/test.TestSlaveQuic.test_file_list_etc_cron_d.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate

software/rapid-cdn/test/test_data/test.TestSlaveQuic.test_file_list_plugin.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/aibcc-sign-promise.py
+T-0/etc/plugin/aibcc-user-caucase-updater.py
+T-0/etc/plugin/aikc-sign-promise.py
+T-0/etc/plugin/aikc-user-caucase-updater.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-backend-haproxy-statistic-url-frontend-node-1.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/master-introspection-server-ip-port-listening.py
+T-0/etc/plugin/master-key-download-url-ready-promise.py
+T-0/etc/plugin/master-key-generate-auth-url-ready-promise.py
+T-0/etc/plugin/master-key-upload-url-ready-promise.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/publish-failsafe-error.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-kedifa-auth-ready.py
+T-1/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr-ip-port-listening.py
+T-2/etc/plugin/frontend-frontend-haproxy-configuration-promise.py
+T-2/etc/plugin/frontend_haproxy_ipv4_http.py
+T-2/etc/plugin/frontend_haproxy_ipv4_https.py
+T-2/etc/plugin/frontend_haproxy_ipv6_http.py
+T-2/etc/plugin/frontend_haproxy_ipv6_https.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-key-download-url-ready.py
+T-2/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py