software/kvm: dcron mock

Thanks to disabling real cron and exposing it's environemnt, it's possible
to provide methods to call cron actions in controllable manner.

software/kvm/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = b6204319cca4264b3c351d4dd1f2b5d0
+md5sum = 7d269cc9da65403476fe95b9975a565e
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2

software/kvm/instance.cfg.in

@@ -80,7 +80,7 @@ extra-context =
   raw curl_executable_location ${curl:location}/bin/curl
   raw dash_executable_location ${dash:location}/bin/dash
   raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
-  raw dcron_executable_location ${dcron:location}/sbin/crond
+  raw dcron_executable_location ${dcron-output:crond}
   raw debian_amd64_netinst_location ${debian-amd64-bullseye-netinst.iso:target}
   raw whitelist_domains_default ${whitelist-domains-default:target}
   raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:output}

software/kvm/test/buildout.hash.cfg

+# THIS IS NOT A BUILDOUT FILE, despite purposedly using a compatible syntax.
+# The only allowed lines here are (regexes):
+# - "^#" comments, copied verbatim
+# - "^[" section beginings, copied verbatim
+# - lines containing an "=" sign which must fit in the following categorie.
+#   - "^\s*filename\s*=\s*path\s*$" where "path" is relative to this file
+#     Copied verbatim.
+#   - "^\s*hashtype\s*=.*" where "hashtype" is one of the values supported
+#     by the re-generation script.
+#     Re-generated.
+# - other lines are copied verbatim
+# Substitution (${...:...}), extension ([buildout] extends = ...) and
+# section inheritance (< = ...) are NOT supported (but you should really
+# not need these here).
+
+[template-test]
+filename = test-instance.cfg.in
+md5sum = 8650b9054e2a1b74b6ceca6927b56b90

software/kvm/test/mock-crond

+#!/bin/sh
+echo $*
+while :; do sleep 5 ; done

software/kvm/test/test-instance.cfg.in

+[buildout]
+extends = ${template:output}
+parts +=
+  cron-env
+  cron-env-service
+  cron-env-entry-environment
+
+
+[directory]
+recipe = slapos.cookbook:mkdirectory
+etc = $${buildout:directory}/etc
+services = $${:etc}/service
+bin = $${buildout:directory}/bin
+var = $${buildout:directory}/var
+log = $${:var}/log
+cron-env-entries = $${:etc}/cron-env.d
+crontabs-env = $${:etc}/crontabs-env
+cronstamps-env = $${:etc}/cronstamps-env
+
+[cron-env]
+recipe = slapos.cookbook:cron
+dcrond-binary = ${dcron-output:crond-orig}
+cron-entries = $${directory:cron-env-entries}
+crontabs = $${directory:crontabs-env}
+cronstamps = $${directory:cronstamps-env}
+catcher = $${cron-env-simplelogger:wrapper}
+binary = $${directory:bin}/crond-env_raw
+
+[cron-env-service]
+recipe = slapos.cookbook:wrapper
+command-line = $${cron-env:binary}
+wrapper-path = $${directory:services}/crond-env
+hash-existing-files = $${buildout:directory}/software_release/buildout.cfg
+
+[cron-env-simplelogger]
+recipe = slapos.cookbook:simplelogger
+wrapper = $${directory:bin}/cron-env_simplelogger
+log = $${directory:log}/crond-env.log
+
+[cron-env-entry-environment]
+<= cron-env
+recipe = slapos.cookbook:cron.d
+name = environment
+frequency = * * * * *
+command = ${buildout:executable} -c 'import os ; import json ; print(json.dumps(dict(os.environ)))' > $${directory:var}/cron-environment.json

software/kvm/test/test-software.cfg

+[buildout]
+extends =
+  ../software.cfg
+  buildout.hash.cfg
+
+parts +=
+  template-test
+
+[template]
+output = ${buildout:directory}/template-original.cfg
+
+[template-test]
+recipe = slapos.recipe.template
+url = ${:_profile_base_location_}/${:filename}
+output = ${buildout:directory}/template.cfg
+
+[dcron-output]
+crond = ${:_profile_base_location_}/mock-crond
+crond-orig = ${dcron:location}/sbin/crond
+

software/kvm/test/test.py

@@ -59,8 +59,7 @@ skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
 
 if has_kvm:
   setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
-    os.path.abspath(
-      os.path.join(os.path.dirname(__file__), '..', 'software.cfg')))
+    os.path.join(os.path.dirname(__file__), 'test-software.cfg'))
   # XXX Keep using slapos node instance --all, because of missing promises
   InstanceTestCase.slap._force_slapos_node_instance_all = True
 else:
@@ -318,6 +317,7 @@ i0:6tunnel-10443-{hash}-on-watch RUNNING
 i0:bootstrap-monitor EXITED
 i0:certificate_authority-{hash}-on-watch RUNNING
 i0:crond-{hash}-on-watch RUNNING
+i0:crond-env-{hash}-on-watch RUNNING
 i0:kvm-{kvm-hash-value}-on-watch RUNNING
 i0:kvm_controller EXITED
 i0:monitor-httpd-{hash}-on-watch RUNNING
@@ -722,6 +722,43 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, KVMTestCase):
     self.assertIn('<title>noVNC</title>', result.text)
 
 
+class CronMixin(object):
+  def setUp(self):
+    super().setUp()
+    # wait until all installed partition have var/cron-environment.json
+    for i in range(20):
+      missing_list = []
+      for installed in glob.glob(os.path.join(
+        self.slap._instance_root, '*', '.installed.cfg')):
+        cron_environment = os.path.join(
+          '/', *installed.split('/')[:-1], 'var', 'cron-environment.json')
+        if not os.path.exists(cron_environment):
+          missing_list.append(cron_environment)
+      if len(missing_list) == 0:
+        break
+      time.sleep(1)
+    else:
+      raise ValueError('Missing cron environment', ' '.join(missing_list))
+
+  @classmethod
+  def executeCronDJob(cls, instance_type, cron):
+    jobpath = cls.getPartitionPath('kvm-export', 'etc', 'cron.d', 'backup')
+    with open(
+      cls.getPartitionPath(
+          'kvm-export', 'var', 'cron-environment.json')) as fh:
+      cron_environment = json.load(fh)
+    job_list = []
+    with open(jobpath, 'r') as fh:
+      for job in fh.readlines():
+        job_list.append(' '.join(job.split(' ')[5:]))
+    job_list_output = []
+    for job in job_list:
+      job_list_output.append(subprocess.run(
+        job, env=cron_environment, shell=True, stdout=subprocess.PIPE,
+        stderr=subprocess.STDOUT))
+    return job_list_output
+
+
 @skipUnlessKvm
 class TestInstanceResilient(KVMTestCase, KvmMixin):
   __partition_reference__ = 'ir'
@@ -795,11 +832,13 @@ class TestInstanceResilient(KVMTestCase, KvmMixin):
       """ir0:bootstrap-monitor EXITED
 ir0:certificate_authority-{hash}-on-watch RUNNING
 ir0:crond-{hash}-on-watch RUNNING
+ir0:crond-env-{hash}-on-watch RUNNING
 ir0:monitor-httpd-{hash}-on-watch RUNNING
 ir0:monitor-httpd-graceful EXITED
 ir1:bootstrap-monitor EXITED
 ir1:certificate_authority-{hash}-on-watch RUNNING
 ir1:crond-{hash}-on-watch RUNNING
+ir1:crond-env-{hash}-on-watch RUNNING
 ir1:equeue-on-watch RUNNING
 ir1:monitor-httpd-{hash}-on-watch RUNNING
 ir1:monitor-httpd-graceful EXITED
@@ -811,6 +850,7 @@ ir2:6tunnel-10443-{hash}-on-watch RUNNING
 ir2:bootstrap-monitor EXITED
 ir2:certificate_authority-{hash}-on-watch RUNNING
 ir2:crond-{hash}-on-watch RUNNING
+ir2:crond-env-{hash}-on-watch RUNNING
 ir2:equeue-on-watch RUNNING
 ir2:kvm-{kvm-hash-value}-on-watch RUNNING
 ir2:kvm_controller EXITED
@@ -827,6 +867,7 @@ ir2:whitelist-firewall-{hash} RUNNING
 ir3:bootstrap-monitor EXITED
 ir3:certificate_authority-{hash}-on-watch RUNNING
 ir3:crond-{hash}-on-watch RUNNING
+ir3:crond-env-{hash}-on-watch RUNNING
 ir3:equeue-on-watch RUNNING
 ir3:monitor-httpd-{hash}-on-watch RUNNING
 ir3:monitor-httpd-graceful EXITED
@@ -2687,6 +2728,7 @@ ihs0:6tunnel-10443-{hash}-on-watch RUNNING
 ihs0:bootstrap-monitor EXITED
 ihs0:certificate_authority-{hash}-on-watch RUNNING
 ihs0:crond-{hash}-on-watch RUNNING
+ihs0:crond-env-{hash}-on-watch RUNNING
 ihs0:http-server-{hash}-on-watch RUNNING
 ihs0:kvm-{kvm-hash-value}-on-watch RUNNING
 ihs0:kvm_controller EXITED