software/*/test: waitForInstance a bit more for softwares with missing promises

Now that slapos node instance no longer use --all, some software release test
revealed some problems with softwares, either missing promises or maybe
something that needs to be handled in slapos.core.
Using --all was wrong and we want to stop doing this now.
This commit is a workaround to keep the test passing, we should not have to
use waitForInstance more than once like this. Something when running slapos
node instance should detect that instance is not ready on the first run.

software/caddy-frontend/test/test.py

@@ -5769,7 +5769,8 @@ class TestSlaveSlapOSMasterCertificateCompatibility(
         partition_reference='custom_domain_ssl_crt_ssl_key_ssl_ca_crt',
         partition_parameter_kw=slave_parameter_dict,
     )
-
+    # XXX run instance twice because of missing promise
+    for _ in range(2):
       self.slap.waitForInstance()
     self.runKedifaUpdater()
     result = fakeHTTPSResult(
@@ -6112,6 +6113,8 @@ class TestSlaveRejectReportUnsafeDamaged(SlaveHttpFrontendTestCase):
     super(TestSlaveRejectReportUnsafeDamaged, cls).setUpClass()
     cls.fillSlaveParameterDictDict()
     cls.requestSlaves()
+    # XXX run instance twice because of missing promise
+    for _ in range(2):
       try:
         cls.slap.waitForInstance(
           max_retry=2  # two runs shall be enough

software/kvm/test/test.py

@@ -53,18 +53,35 @@ has_kvm = os.access('/dev/kvm', os.R_OK | os.W_OK)
 skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')
 
 if has_kvm:
-  setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
+  setUpModule, _InstanceTestCase = makeModuleSetUpAndTestCaseClass(
     os.path.abspath(
       os.path.join(os.path.dirname(__file__), '..',
                    'software%s.cfg' % ("-py3" if six.PY3 else ""))))
 else:
-  setUpModule, InstanceTestCase = None, unittest.TestCase
+  setUpModule, _InstanceTestCase = None, unittest.TestCase
 
   class SanityCheckTestCase(unittest.TestCase):
     def test_kvm_sanity_check(self):
       self.fail('This environment is not usable for kvm testing,'
                 ' as it lacks kvm_intel kernel module')
 
+
+class InstanceTestCase(_InstanceTestCase):
+  def waitForInstance(self, max_retry):
+    """Wait for instance to be processed, with workarounds.
+
+    This works around missing promises that cause `slapos node instance` to
+    exit with success code too early, when instance is not completly processed
+    yet.
+    The workaround consists in waiting for `slapos node instance` to exit with
+    success code several times. Ideally, `slapos node instance` should exit with
+    error code until the instance is fully ready and this wrapper should not be
+    necessary, `self.slap.waitForInstance()` should be enough.
+    """
+    for _ in range(3):
+      self.slap.waitForInstance(max_retry)
+
+
 bootstrap_common_param_dict = {
     # the bootstrap script is vm-bootstrap
     "bootstrap-script-url":
@@ -129,6 +146,7 @@ class KvmMixin(object):
 
   def raising_waitForInstance(self, max_retry):
     with self.assertRaises(SlapOSNodeCommandError):
+      for _ in range(3): # XXX workaround missing promises
         self.slap.waitForInstance(max_retry=max_retry)
 
   def rerequestInstance(self, parameter_dict, state='started'):
@@ -686,10 +704,10 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     # clean up the instance for other tests
     # 1st remove all images...
     self.rerequestInstance({self.key: ''})
-    self.slap.waitForInstance(max_retry=10)
+    self.waitForInstance(max_retry=10)
     # 2nd ...move instance to "default" state
     self.rerequestInstance({})
-    self.slap.waitForInstance(max_retry=10)
+    self.waitForInstance(max_retry=10)
     self.stopImageHttpServer()
     super(InstanceTestCase, self).tearDown()
 
@@ -700,7 +718,7 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
         self.fake_image2_md5sum)
     }
     self.rerequestInstance(partition_parameter_kw)
-    self.slap.waitForInstance(max_retry=10)
+    self.waitForInstance(max_retry=10)
     # check that image is correctly downloaded and linked
     kvm_instance_partition = os.path.join(
       self.slap.instance_directory, self.kvm_instance_partition_reference)
@@ -744,9 +762,9 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     # mimic the requirement: restart the instance by requesting it stopped and
     # then started started, like user have to do it
     self.rerequestInstance(partition_parameter_kw, state='stopped')
-    self.slap.waitForInstance(max_retry=1)
+    self.waitForInstance(max_retry=1)
     self.rerequestInstance(partition_parameter_kw, state='started')
-    self.slap.waitForInstance(max_retry=3)
+    self.waitForInstance(max_retry=3)
 
     self.assertEqual(
       [
@@ -762,7 +780,7 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     # reflected
     partition_parameter_kw[self.key] = ''
     self.rerequestInstance(partition_parameter_kw)
-    self.slap.waitForInstance(max_retry=2)
+    self.waitForInstance(max_retry=2)
     self.assertEqual(
       os.listdir(image_repository),
       []
@@ -771,9 +789,9 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
     # mimic the requirement: restart the instance by requesting it stopped and
     # then started started, like user have to do it
     self.rerequestInstance(partition_parameter_kw, state='stopped')
-    self.slap.waitForInstance(max_retry=1)
+    self.waitForInstance(max_retry=1)
     self.rerequestInstance(partition_parameter_kw, state='started')
-    self.slap.waitForInstance(max_retry=3)
+    self.waitForInstance(max_retry=3)
 
     # again only default image is available in the running process
     self.assertEqual(
@@ -894,7 +912,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
         self.fake_image, self.fake_image_md5sum)
     }
     self.rerequestInstance(partition_parameter_kw)
-    self.slap.waitForInstance(max_retry=10)
+    self.waitForInstance(max_retry=10)
     # check that image is correctly downloaded and linked
     for image_directory in [
       'boot-image-url-list-repository', 'boot-image-url-select-repository']:
@@ -933,9 +951,9 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # mimic the requirement: restart the instance by requesting it stopped and
     # then started started, like user have to do it
     self.rerequestInstance(partition_parameter_kw, state='stopped')
-    self.slap.waitForInstance(max_retry=1)
+    self.waitForInstance(max_retry=1)
     self.rerequestInstance(partition_parameter_kw, state='started')
-    self.slap.waitForInstance(max_retry=3)
+    self.waitForInstance(max_retry=3)
 
     self.assertEqual(
       [
@@ -951,7 +969,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # reflected
     self.rerequestInstance(
       {'boot-image-url-list': '', 'boot-image-url-select': ''})
-    self.slap.waitForInstance(max_retry=2)
+    self.waitForInstance(max_retry=2)
     for image_directory in [
       'boot-image-url-list-repository', 'boot-image-url-select-repository']:
       image_repository = os.path.join(
@@ -966,7 +984,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     partition_parameter_kw[self.key] = ''
     partition_parameter_kw['boot-image-url-list'] = ''
     self.rerequestInstance(partition_parameter_kw)
-    self.slap.waitForInstance(max_retry=2)
+    self.waitForInstance(max_retry=2)
     self.assertEqual(
       os.listdir(image_repository),
       []
@@ -975,9 +993,9 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
     # mimic the requirement: restart the instance by requesting it stopped and
     # then started started, like user have to do it
     self.rerequestInstance(partition_parameter_kw, state='stopped')
-    self.slap.waitForInstance(max_retry=1)
+    self.waitForInstance(max_retry=1)
     self.rerequestInstance(partition_parameter_kw, state='started')
-    self.slap.waitForInstance(max_retry=3)
+    self.waitForInstance(max_retry=3)
 
     # again only default image is available in the running process
     self.assertEqual(
@@ -1045,7 +1063,7 @@ class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
         }
       }
     })})
-    self.slap.waitForInstance(max_retry=10)
+    self.waitForInstance(max_retry=10)
     KVM0_config = os.path.join(
       self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
       self.config_file_name)

software/monitor/test/test.py

@@ -233,6 +233,19 @@ class EdgeSlaveMixin(MonitorTestMixin):
       )
 
   def initiateSurykatkaRun(self):
+    # XXX we run twice as a workaround for the fact that after requesting new
+    # shared instances we don't have promise to wait for the processing of
+    # these shared instances to be completed.
+    # The sequence is something like this:
+    #   - `requestEdgetestSlaves` will request edgetest partition
+    #   - first `waitForInstance` will process the edgetest partition, which will
+    #     request a edgebot partition, but without promise to wait for the
+    #     processing to be finished, so the first run of `slapos node instance`
+    #     exits with success code and `waitForInstance` return.
+    #   - second `waitForInstance` process the edgebot partition.
+    # Once we implement a promise (or something similar) here, we should not
+    # have to run twice.
+    for _ in range(2):
       try:
         self.slap.waitForInstance(max_retry=2)
       except Exception: