caddy-frontend: Cover manual CSR handling

software/caddy-frontend/test/test.py

@@ -53,6 +53,9 @@ import sys
 import logging
 import random
 import string
+from slapos.slap.standalone import SlapOSNodeInstanceError
+import caucase.client
+import caucase.utils
 
 
 try:
@@ -741,25 +744,41 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
       self.logger.warning(
         'Process %s still alive' % (self.server_https_auth_process, ))
 
+  @classmethod
+  def _fetchKedifaCaucaseCaCertificateFile(cls, parameter_dict):
+    ca_certificate = requests.get(
+      parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
+    assert ca_certificate.status_code == httplib.OK
+    cls.kedifa_caucase_ca_certificate_file = os.path.join(
+      cls.working_directory, 'kedifa-caucase.ca.crt.pem')
+    open(cls.kedifa_caucase_ca_certificate_file, 'w').write(
+        ca_certificate.text)
+
+  @classmethod
+  def _fetchBackendClientCaCertificateFile(cls, parameter_dict):
+    ca_certificate = requests.get(
+      parameter_dict['backend-client-caucase-url'] + '/cas/crt/ca.crt.pem')
+    assert ca_certificate.status_code == httplib.OK
+    cls.backend_client_caucase_ca_certificate_file = os.path.join(
+      cls.working_directory, 'backend-client-caucase.ca.crt.pem')
+    open(cls.backend_client_caucase_ca_certificate_file, 'w').write(
+      ca_certificate.text)
+
   @classmethod
   def setUpMaster(cls):
     # run partition until AIKC finishes
     cls.runComputerPartitionUntil(
       cls.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
     parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
-    ca_certificate = requests.get(
-      parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
-    assert ca_certificate.status_code == httplib.OK
-    cls.ca_certificate_file = os.path.join(cls.working_directory, 'ca.crt.pem')
-    open(cls.ca_certificate_file, 'w').write(ca_certificate.text)
+    cls._fetchKedifaCaucaseCaCertificateFile(parameter_dict)
     auth = requests.get(
       parameter_dict['master-key-generate-auth-url'],
-      verify=cls.ca_certificate_file)
+      verify=cls.kedifa_caucase_ca_certificate_file)
     assert auth.status_code == httplib.CREATED
     upload = requests.put(
       parameter_dict['master-key-upload-url'] + auth.text,
       data=cls.key_pem + cls.certificate_pem,
-      verify=cls.ca_certificate_file)
+      verify=cls.kedifa_caucase_ca_certificate_file)
     assert upload.status_code == httplib.CREATED
     cls.runKedifaUpdater()
 
@@ -1063,6 +1082,17 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     cls.stopServerProcess()
     super(HttpFrontendTestCase, cls)._cleanup(snapshot_name)
 
+  @classmethod
+  def _workingDirectorySetUp(cls):
+      # do working directory
+      cls.working_directory = os.path.join(os.path.realpath(
+          os.environ.get(
+              'SLAPOS_TEST_WORKING_DIR',
+              os.path.join(os.getcwd(), '.slapos'))),
+          'caddy-frontend-test')
+      if not os.path.isdir(cls.working_directory):
+        os.mkdir(cls.working_directory)
+
   @classmethod
   def setUpClass(cls):
     try:
@@ -1084,19 +1114,12 @@ class HttpFrontendTestCase(SlapOSInstanceTestCase):
     super(HttpFrontendTestCase, cls).setUpClass()
 
     try:
+      cls._workingDirectorySetUp()
       # expose instance directory
       cls.instance_path = cls.slap.instance_directory
       # expose software directory, extract from found computer partition
       cls.software_path = os.path.realpath(os.path.join(
           cls.computer_partition_root_path, 'software_release'))
-      # do working directory
-      cls.working_directory = os.path.join(os.path.realpath(
-          os.environ.get(
-              'SLAPOS_TEST_WORKING_DIR',
-              os.path.join(os.getcwd(), '.slapos'))),
-          'caddy-frontend-test')
-      if not os.path.isdir(cls.working_directory):
-        os.mkdir(cls.working_directory)
       cls.setUpMaster()
       cls.waitForCaddy()
     except BaseException:
@@ -1335,6 +1358,120 @@ class TestMasterRequest(HttpFrontendTestCase, TestDataMixin):
     )
 
 
+class TestMasterAIKCDisabledAIBCCDisabledRequest(
+  HttpFrontendTestCase, TestDataMixin):
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'port': HTTPS_PORT,
+      'plain_http_port': HTTP_PORT,
+      'kedifa_port': KEDIFA_PORT,
+      'caucase_port': CAUCASE_PORT,
+      'automatic-internal-kedifa-caucase-csr': 'false',
+      'automatic-internal-backend-client-caucase-csr': 'false',
+    }
+
+  @classmethod
+  def _setUpClass(cls):
+    instance_max_retry = cls.instance_max_retry
+    try:
+      cls.instance_max_retry = 3
+      super(TestMasterAIKCDisabledAIBCCDisabledRequest, cls)._setUpClass()
+    except SlapOSNodeInstanceError:  # Note: SLAPOS_TEST_DEBUG=1 will interrupt
+      pass
+    else:
+      raise ValueError('_setUpClass unexpected success')
+    # Cluster requested without automatic certificate handling will never
+    # stabilize, as nodes can't join to the cluster, so the user is required
+    # to first manually create key and certificate for himself, then manually
+    # create certificates for services
+    cls._workingDirectorySetUp()
+    _, kedifa_key_pem, _, kedifa_csr_pem = createCSR('Kedifa User')
+    _, backend_client_key_pem, _, backend_client_csr_pem = createCSR(
+      'Backend Client User')
+    parameter_dict = cls.requestDefaultInstance(
+      ).getConnectionParameterDict()
+    cls._fetchKedifaCaucaseCaCertificateFile(parameter_dict)
+    cls._fetchBackendClientCaCertificateFile(parameter_dict)
+    with open(cls.kedifa_caucase_ca_certificate_file) as fh:
+      kedifa_ca_pem = fh.read()
+    with open(cls.backend_client_caucase_ca_certificate_file) as fh:
+      backend_client_ca_pem = fh.read()
+
+    kedifa_caucase_url = parameter_dict['kedifa-caucase-url']
+    backend_client_caucase_url = parameter_dict['backend-client-caucase-url']
+
+    # Simulate human: create user keys
+    def getCauCertificate(ca_url, ca_pem, csr_pem):
+      cau_client = caucase.client.CaucaseClient(
+        ca_url=ca_url + '/cau',
+        ca_crt_pem_list=caucase.utils.getCertList(ca_pem),
+      )
+      csr_id = cau_client.createCertificateSigningRequest(csr_pem)
+      return cau_client.getCertificate(csr_id)
+
+    kedifa_crt_pem = getCauCertificate(
+      kedifa_caucase_url, kedifa_ca_pem, kedifa_csr_pem)
+    backend_client_crt_pem = getCauCertificate(
+      backend_client_caucase_url, backend_client_ca_pem,
+      backend_client_csr_pem)
+    kedifa_key_file = os.path.join(cls.working_directory, 'kedifa-key.pem')
+    with open(kedifa_key_file, 'w') as fh:
+      fh.write(kedifa_crt_pem + kedifa_key_pem)
+    backend_client_key_file = os.path.join(
+      cls.working_directory, 'backend-client-key.pem')
+    with open(backend_client_key_file, 'w') as fh:
+      fh.write(backend_client_crt_pem + backend_client_key_pem)
+
+    # Simulate human: create service keys
+    def signAllCasCsr(ca_url, ca_pem, user_key, pending_csr_amount):
+      client = caucase.client.CaucaseClient(
+        ca_url=ca_url + '/cas',
+        ca_crt_pem_list=caucase.utils.getCertList(ca_pem), user_key=user_key)
+      pending_csr_list = client.getPendingCertificateRequestList()
+      assert len(pending_csr_list) == pending_csr_amount
+      for csr_entry in pending_csr_list:
+        client.createCertificate(int(csr_entry['id']))
+
+    signAllCasCsr(kedifa_caucase_url, kedifa_ca_pem, kedifa_key_file, 2)
+    signAllCasCsr(
+      backend_client_caucase_url, backend_client_ca_pem,
+      backend_client_key_file, 1)
+    # Continue instance processing, copy&paste from
+    # slapos.testing.testcase.SlapOSInstanceTestCase._setUpClass
+    # as we hack a lot
+    cls.instance_max_retry = instance_max_retry
+    cls.waitForInstance()
+    cls.computer_partition = cls.requestDefaultInstance()
+    cls.computer_partition_root_path = os.path.join(
+      cls.slap._instance_root, cls.computer_partition.getId())
+
+  def test(self):
+    parameter_dict = self.parseConnectionParameterDict()
+    self.assertKeyWithPop('monitor-setup-url', parameter_dict)
+    self.assertBackendHaproxyStatisticUrl(parameter_dict)
+    self.assertKedifaKeysWithPop(parameter_dict, 'master-')
+    self.assertRejectedSlavePromiseEmptyWithPop(parameter_dict)
+    self.assertKeyWithPop('kedifa-csr-certificate', parameter_dict)
+    self.assertKeyWithPop('kedifa-csr-url', parameter_dict)
+    self.assertKeyWithPop('caddy-frontend-1-kedifa-csr-url', parameter_dict)
+    self.assertKeyWithPop(
+      'caddy-frontend-1-backend-client-csr-url', parameter_dict)
+    self.assertKeyWithPop(
+      'caddy-frontend-1-csr-certificate', parameter_dict)
+    self.assertEqual(
+      {
+        'monitor-base-url': 'https://[%s]:8401' % self._ipv6_address,
+        'backend-client-caucase-url': 'http://[%s]:8990' % self._ipv6_address,
+        'domain': 'None',
+        'accepted-slave-amount': '0',
+        'rejected-slave-amount': '0',
+        'slave-amount': '0',
+        'rejected-slave-dict': {}},
+      parameter_dict
+    )
+
+
 class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
   @classmethod
   def getInstanceParameterDict(cls):
@@ -2535,7 +2672,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
 
     data = self.customdomain_ca_certificate_pem + \
@@ -2545,7 +2682,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
     self.runKedifaUpdater()
 
@@ -2585,7 +2722,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
 
     data = self.ca.certificate_pem
@@ -2593,7 +2730,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
 
     self.assertEqual(httplib.UNPROCESSABLE_ENTITY, upload.status_code)
     self.assertEqual('Key incorrect', upload.text)
@@ -2618,7 +2755,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
 
     _, ca_key_pem, csr, _ = createCSR(
@@ -2629,7 +2766,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
 
     self.assertEqual(httplib.CREATED, upload.status_code)
     self.runKedifaUpdater()
@@ -2671,7 +2808,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
 
     data = self.certificate_pem + self.key_pem + self.ca.certificate_pem
@@ -2679,7 +2816,7 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
 
     self.assertEqual(httplib.CREATED, upload.status_code)
     self.runKedifaUpdater()
@@ -2832,14 +2969,14 @@ class TestSlave(SlaveHttpFrontendTestCase, TestDataMixin):
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
     data = self.customdomain_certificate_pem + \
         self.customdomain_key_pem
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
     self.runKedifaUpdater()
 
@@ -5057,11 +5194,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster(
       cls.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
 
     parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
-    ca_certificate = requests.get(
-      parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
-    assert ca_certificate.status_code == httplib.OK
-    cls.ca_certificate_file = os.path.join(cls.working_directory, 'ca.crt.pem')
-    open(cls.ca_certificate_file, 'w').write(ca_certificate.text)
+    cls._fetchKedifaCaucaseCaCertificateFile(parameter_dict)
     # Do not upload certificates for the master partition
 
   @classmethod
@@ -5120,11 +5253,11 @@ class TestSlaveSlapOSMasterCertificateCompatibilityOverrideMaster(
         self.requestDefaultInstance().getConnectionParameterDict()
     auth = requests.get(
       master_parameter_dict['master-key-generate-auth-url'],
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     requests.put(
       master_parameter_dict['master-key-upload-url'] + auth.text,
       data=key_pem + certificate_pem,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.runKedifaUpdater()
 
     result = fakeHTTPSResult(
@@ -5147,11 +5280,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
       cls.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
 
     parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
-    ca_certificate = requests.get(
-      parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
-    assert ca_certificate.status_code == httplib.OK
-    cls.ca_certificate_file = os.path.join(cls.working_directory, 'ca.crt.pem')
-    open(cls.ca_certificate_file, 'w').write(ca_certificate.text)
+    cls._fetchKedifaCaucaseCaCertificateFile(parameter_dict)
     # Do not upload certificates for the master partition
 
   @classmethod
@@ -5413,7 +5542,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
 
     data = certificate_pem + key_pem
@@ -5421,7 +5550,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
     self.runKedifaUpdater()
 
@@ -5504,7 +5633,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
 
     data = certificate_pem + key_pem
@@ -5512,7 +5641,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
 
     self.runKedifaUpdater()
@@ -5588,7 +5717,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
 
     data = certificate_pem + key_pem
@@ -5596,7 +5725,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
 
     self.runKedifaUpdater()
@@ -5681,7 +5810,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     # as now the place to put the key is known put the key there
     auth = requests.get(
       generate_auth,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, auth.status_code)
 
     data = certificate_pem + key_pem
@@ -5689,7 +5818,7 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
     upload = requests.put(
       upload_url + auth.text,
       data=data,
-      verify=self.ca_certificate_file)
+      verify=self.kedifa_caucase_ca_certificate_file)
     self.assertEqual(httplib.CREATED, upload.status_code)
 
     self.runKedifaUpdater()
@@ -5907,11 +6036,7 @@ class TestSlaveSlapOSMasterCertificateCompatibilityUpdate(
       cls.untilNotReadyYetNotInMasterKeyGenerateAuthUrl)
 
     parameter_dict = cls.requestDefaultInstance().getConnectionParameterDict()
-    ca_certificate = requests.get(
-      parameter_dict['kedifa-caucase-url'] + '/cas/crt/ca.crt.pem')
-    assert ca_certificate.status_code == httplib.OK
-    cls.ca_certificate_file = os.path.join(cls.working_directory, 'ca.crt.pem')
-    open(cls.ca_certificate_file, 'w').write(ca_certificate.text)
+    cls._fetchKedifaCaucaseCaCertificateFile(parameter_dict)
     # Do not upload certificates for the master partition
 
   instance_parameter_dict = {

software/caddy-frontend/test/test_data/test.TestMasterAIKCDisabledAIBCCDisabledRequest.test_file_list_etc_cron_d-CADDY.txt

+T-0/etc/cron.d/logrotate
+T-0/etc/cron.d/monitor-configurator
+T-0/etc/cron.d/monitor-globalstate
+T-0/etc/cron.d/monitor_collect
+T-1/etc/cron.d/logrotate
+T-1/etc/cron.d/monitor-configurator
+T-1/etc/cron.d/monitor-globalstate
+T-1/etc/cron.d/monitor_collect
+T-2/etc/cron.d/logrotate
+T-2/etc/cron.d/monitor-configurator
+T-2/etc/cron.d/monitor-globalstate
+T-2/etc/cron.d/monitor_collect
+T-2/etc/cron.d/trafficserver-logrotate

software/caddy-frontend/test/test_data/test.TestMasterAIKCDisabledAIBCCDisabledRequest.test_file_list_log-CADDY.txt

+T-0/var/log/monitor-httpd-access.log
+T-0/var/log/monitor-httpd-error.log
+T-0/var/log/slapgrid-T-0-error.log
+T-1/var/log/expose-csr.log
+T-1/var/log/kedifa.log
+T-1/var/log/monitor-httpd-access.log
+T-1/var/log/monitor-httpd-error.log
+T-2/var/log/backend-haproxy.log
+T-2/var/log/expose-csr.log
+T-2/var/log/frontend-access.log
+T-2/var/log/frontend-error.log
+T-2/var/log/monitor-httpd-access.log
+T-2/var/log/monitor-httpd-error.log
+T-2/var/log/slave-introspection-access.log
+T-2/var/log/slave-introspection-error.log
+T-2/var/log/trafficserver/manager.log

software/caddy-frontend/test/test_data/test.TestMasterAIKCDisabledAIBCCDisabledRequest.test_file_list_plugin-CADDY.txt

+T-0/etc/plugin/__init__.py
+T-0/etc/plugin/buildout-T-0-status.py
+T-0/etc/plugin/caucased-backend-client.py
+T-0/etc/plugin/check-backend-haproxy-statistic-url-caddy-frontend-1.py
+T-0/etc/plugin/check-free-disk-space.py
+T-0/etc/plugin/monitor-bootstrap-status.py
+T-0/etc/plugin/monitor-http-frontend.py
+T-0/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-0/etc/plugin/rejected-slave-publish-ip-port-listening.py
+T-0/etc/plugin/rejected-slave.py
+T-1/etc/plugin/__init__.py
+T-1/etc/plugin/buildout-T-1-status.py
+T-1/etc/plugin/caucased.py
+T-1/etc/plugin/check-free-disk-space.py
+T-1/etc/plugin/expose-csr-ip-port-listening.py
+T-1/etc/plugin/kedifa-http-reply.py
+T-1/etc/plugin/monitor-bootstrap-status.py
+T-1/etc/plugin/monitor-http-frontend.py
+T-1/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-1/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/__init__.py
+T-2/etc/plugin/backend-client-caucase-updater.py
+T-2/etc/plugin/backend-haproxy-configuration.py
+T-2/etc/plugin/backend-haproxy-statistic-frontend.py
+T-2/etc/plugin/backend_haproxy_http.py
+T-2/etc/plugin/backend_haproxy_https.py
+T-2/etc/plugin/buildout-T-2-status.py
+T-2/etc/plugin/caddy_frontend_ipv4_http.py
+T-2/etc/plugin/caddy_frontend_ipv4_https.py
+T-2/etc/plugin/caddy_frontend_ipv6_http.py
+T-2/etc/plugin/caddy_frontend_ipv6_https.py
+T-2/etc/plugin/caucase-updater.py
+T-2/etc/plugin/check-free-disk-space.py
+T-2/etc/plugin/expose-csr-ip-port-listening.py
+T-2/etc/plugin/frontend-caddy-configuration-promise.py
+T-2/etc/plugin/monitor-bootstrap-status.py
+T-2/etc/plugin/monitor-http-frontend.py
+T-2/etc/plugin/monitor-httpd-listening-on-tcp.py
+T-2/etc/plugin/promise-logrotate-setup.py
+T-2/etc/plugin/re6st-connectivity.py
+T-2/etc/plugin/slave-introspection-configuration.py
+T-2/etc/plugin/slave_introspection_https.py
+T-2/etc/plugin/trafficserver-cache-availability.py
+T-2/etc/plugin/trafficserver-port-listening.py

software/caddy-frontend/test/test_data/test.TestMasterAIKCDisabledAIBCCDisabledRequest.test_file_list_run-CADDY.txt

+T-0/var/run/monitor-httpd.pid
+T-1/var/run/kedifa.pid
+T-1/var/run/monitor-httpd.pid
+T-2/var/run/backend-haproxy-rsyslogd.pid
+T-2/var/run/backend-haproxy.pid
+T-2/var/run/backend_haproxy_configuration_last_state
+T-2/var/run/backend_haproxy_graceful_configuration_state_signature
+T-2/var/run/bhlog.sck
+T-2/var/run/graceful_configuration_state_signature
+T-2/var/run/httpd.pid
+T-2/var/run/monitor-httpd.pid
+T-2/var/run/slave-introspection.pid
+T-2/var/run/slave_introspection_configuration_last_state
+T-2/var/run/slave_introspection_graceful_configuration_state_signature

software/caddy-frontend/test/test_data/test.TestMasterAIKCDisabledAIBCCDisabledRequest.test_supervisor_state-CADDY.txt

+T-0:bootstrap-monitor EXITED
+T-0:caucased-backend-client-{hash-generic}-on-watch RUNNING
+T-0:certificate_authority-{hash-generic}-on-watch RUNNING
+T-0:crond-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-0:monitor-httpd-graceful EXITED
+T-0:rejected-slave-publish-{hash-rejected-slave-publish}-on-watch RUNNING
+T-1:bootstrap-monitor EXITED
+T-1:caucase-updater-on-watch RUNNING
+T-1:caucased-{hash-generic}-on-watch RUNNING
+T-1:certificate_authority-{hash-generic}-on-watch RUNNING
+T-1:crond-{hash-generic}-on-watch RUNNING
+T-1:expose-csr-{hash-generic}-on-watch RUNNING
+T-1:kedifa-{hash-generic}-on-watch RUNNING
+T-1:kedifa-reloader EXITED
+T-1:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-1:monitor-httpd-graceful EXITED
+T-2:6tunnel-11080-{hash-generic}-on-watch RUNNING
+T-2:6tunnel-11443-{hash-generic}-on-watch RUNNING
+T-2:backend-client-login-certificate-caucase-updater-on-watch RUNNING
+T-2:backend-haproxy-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-rsyslogd-{hash-generic}-on-watch RUNNING
+T-2:backend-haproxy-safe-graceful EXITED
+T-2:bootstrap-monitor EXITED
+T-2:certificate_authority-{hash-generic}-on-watch RUNNING
+T-2:crond-{hash-generic}-on-watch RUNNING
+T-2:expose-csr-{hash-generic}-on-watch RUNNING
+T-2:frontend-caddy-safe-graceful EXITED
+T-2:frontend_caddy-{hash-caddy-T-2}-on-watch RUNNING
+T-2:kedifa-login-certificate-caucase-updater-on-watch RUNNING
+T-2:kedifa-updater-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-{hash-generic}-on-watch RUNNING
+T-2:monitor-httpd-graceful EXITED
+T-2:slave-instrospection-nginx-{hash-generic}-on-watch RUNNING
+T-2:slave-introspection-safe-graceful EXITED
+T-2:trafficserver-{hash-generic}-on-watch RUNNING
+T-2:trafficserver-reload EXITED