Update Release Candidate

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 7e90da1f6dac4233e1aa3248f48e357c
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 44dc93281f2fffe64d014754fae1b38a
+md5sum = ff9fb2378a48d1ca8c72f6a87d3a0221
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in

software/kvm/instance-kvm.cfg.jinja2

@@ -939,7 +939,9 @@ recipe = collective.recipe.template
 input = inline:
   {{ data_list | join('\n  ') }}
 output = {{ file_path }}
+{%   if mode %}
 mode = {{ mode }}
+{%   endif %}
 {% endmacro -%}
 
 # write vm-data into file public/data
@@ -1110,7 +1112,7 @@ data-to-vm =
 {% set key_list =  v.split('\n') -%}
 {{ k }} =
   {{ key_list | join('\n  ') }}
-{% elif k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains'] %}
+{% elif k in ['boot-image-url-list', 'boot-image-url-select', 'whitelist-domains', 'data-to-vm'] %}
 {# needs to decorate possibly multiline or maybe unsafe value #}
 {{ k }} = {{ dumps(v) }}
 {% else -%}

software/kvm/test/test.py

@@ -221,6 +221,12 @@ i0:whitelist-firewall-{hash} RUNNING""",
     )
 
 
+@skipUnlessKvm
+class TestInstanceJson(
+  KvmMixinJson, TestInstance):
+  pass
+
+
 @skipUnlessKvm
 class TestMemoryManagement(InstanceTestCase, KvmMixin):
   __partition_reference__ = 'i'
@@ -817,7 +823,7 @@ class TestInstanceNbdServerJson(
   pass
 
 
-class FakeImageHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
+class HttpHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
   def log_message(self, *args):
     if os.environ.get('SLAPOS_TEST_DEBUG'):
       return SimpleHTTPServer.SimpleHTTPRequestHandler.log_message(self, *args)
@@ -831,7 +837,7 @@ class FakeImageServerMixin(KvmMixin):
     cls.image_source_directory = tempfile.mkdtemp()
     server = SocketServer.TCPServer(
       (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address)),
-      FakeImageHandler)
+      HttpHandler)
 
     # c89f17758be13adeb06886ef935d5ff1
     fake_image_content = b'fake_image_content'
@@ -2195,3 +2201,125 @@ class TestExternalDiskModernIndexRequired(InstanceTestCase, ExternalDiskMixin):
         self.getExternalDiskInstanceParameterDict(
           self.first_disk, second_disk, self.third_disk))})
     self.raising_waitForInstance(10)
+
+
+@skipUnlessKvm
+class TestInstanceHttpServer(InstanceTestCase, KvmMixin):
+  __partition_reference__ = 'ihs'
+
+  @classmethod
+  def startHttpServer(cls):
+    cls.http_directory = tempfile.mkdtemp()
+    server = SocketServer.TCPServer(
+      (cls._ipv4_address, findFreeTCPPort(cls._ipv4_address)),
+      HttpHandler)
+
+    bootstrap_script = b'bootstrap_script'
+    cls.bootstrap_script_md5sum = hashlib.md5(bootstrap_script).hexdigest()
+    with open(os.path.join(
+      cls.http_directory, cls.bootstrap_script_md5sum), 'wb') as fh:
+      fh.write(bootstrap_script)
+
+    url = 'http://%s:%s' % server.server_address
+    cls.bootstrap_script_url = '/'.join([url, cls.bootstrap_script_md5sum])
+
+    old_dir = os.path.realpath(os.curdir)
+    os.chdir(cls.http_directory)
+    try:
+      cls.server_process = multiprocessing.Process(
+        target=server.serve_forever, name='HttpServer')
+      cls.server_process.start()
+    finally:
+      os.chdir(old_dir)
+
+  @classmethod
+  def stopHttpServer(cls):
+    cls.logger.debug('Stopping process %s' % (cls.server_process,))
+    cls.server_process.join(10)
+    cls.server_process.terminate()
+    time.sleep(0.1)
+    if cls.server_process.is_alive():
+      cls.logger.warning(
+        'Process %s still alive' % (cls.server_process, ))
+
+    shutil.rmtree(cls.http_directory)
+
+  @classmethod
+  def setUpClass(cls):
+    cls.startHttpServer()
+    super(TestInstanceHttpServer, cls).setUpClass()
+
+  @classmethod
+  def tearDownClass(cls):
+    super(TestInstanceHttpServer, cls).tearDownClass()
+    cls.stopHttpServer()
+
+  @classmethod
+  def getInstanceParameterDict(cls):
+    return {
+      'enable-http-server': True,
+      'bootstrap-script-url': '%s#%s' % (
+        cls.bootstrap_script_url, cls.bootstrap_script_md5sum),
+      'data-to-vm': """data
+to
+vm""",
+    }
+
+  def test(self):
+    connection_parameter_dict = self.getConnectionParameterDictJson()
+    present_key_list = []
+    assert_key_list = [
+     'backend-url', 'url', 'monitor-setup-url', 'ipv6-network-info',
+     'tap-ipv4', 'tap-ipv6']
+    for k in assert_key_list:
+      if k in connection_parameter_dict:
+        present_key_list.append(k)
+        connection_parameter_dict.pop(k)
+    self.assertEqual(
+      connection_parameter_dict,
+      {
+        'ipv6': self._ipv6_address,
+        'maximum-extra-disk-amount': '0',
+        'monitor-base-url': 'https://[%s]:8026' % (self._ipv6_address,),
+        'nat-rule-port-tcp-22': '%s : 10022' % (self._ipv6_address,),
+        'nat-rule-port-tcp-443': '%s : 10443' % (self._ipv6_address,),
+        'nat-rule-port-tcp-80': '%s : 10080' % (self._ipv6_address,),
+      }
+    )
+    self.assertEqual(set(present_key_list), set(assert_key_list))
+    self.assertEqual(
+      """ihs0:6tunnel-10022-{hash}-on-watch RUNNING
+ihs0:6tunnel-10080-{hash}-on-watch RUNNING
+ihs0:6tunnel-10443-{hash}-on-watch RUNNING
+ihs0:bootstrap-monitor EXITED
+ihs0:certificate_authority-{hash}-on-watch RUNNING
+ihs0:crond-{hash}-on-watch RUNNING
+ihs0:http-server-{hash}-on-watch RUNNING
+ihs0:kvm-{kvm-hash-value}-on-watch RUNNING
+ihs0:kvm_controller EXITED
+ihs0:monitor-httpd-{hash}-on-watch RUNNING
+ihs0:monitor-httpd-graceful EXITED
+ihs0:websockify-{hash}-on-watch RUNNING
+ihs0:whitelist-domains-download-{hash} RUNNING
+ihs0:whitelist-firewall-{hash} RUNNING""",
+      self.getProcessInfo()
+    )
+    public_dir = os.path.join(
+      self.computer_partition_root_path, 'srv', 'public')
+    self.assertEqual(
+      ['data', 'gateway', 'hostname', 'ipv4',
+       'ipv6_config.sh', 'netmask', 'network', 'vm-bootstrap'],
+      sorted(os.listdir(public_dir))
+    )
+    with open(os.path.join(public_dir, 'data'), 'r') as fh:
+      self.assertEqual("""data
+to
+vm""", fh.read())
+    with open(os.path.join(public_dir, 'vm-bootstrap'), 'r') as fh:
+      self.assertEqual('bootstrap_script', fh.read())
+
+
+@skipUnlessKvm
+class TestInstanceHttpServerJson(
+  KvmMixinJson, TestInstanceHttpServer):
+  pass

software/monitor/buildout.hash.cfg

@@ -26,7 +26,7 @@ md5sum = 2eb5596544d9c341acf653d4f7ce2680
 
 [template-monitor-edgetest-basic]
 _update_hash_filename_ = instance-monitor-edgetest-basic.cfg.jinja2
-md5sum = 8e0c4041f680312ff054687d7f28275a
+md5sum = efc528296ddf7fade335d5f4241c1828
 
 [network-bench-cfg]
 filename = network_bench.cfg.in

software/monitor/instance-monitor-edgetest-basic.cfg.jinja2

@@ -14,7 +14,7 @@
 {%-   endif %}
 {%- endfor %}
 {%- set CHECK_DICT = {} %}
-{%- for check_name, check_definition in slapparameter_dict['check-dict'].items() %}
+{%- for check_name, check_definition in slapparameter_dict.get('check-dict', {}).items() %}
 {%-   if 'url-list' in check_definition %}
 {%-     set check = DEFAULT_DICT.copy()  %}
 {%-     set url_list = [] %}

software/monitor/software.cfg

@@ -62,7 +62,7 @@ scripts =
 
 [versions]
 
-surykatka = 0.7.1
+surykatka = 0.8.0
 
 # For surykatka 0.7.1
 click = 8.0.1

stack/erp5/buildout.hash.cfg

@@ -30,7 +30,7 @@ md5sum = cee995829fbd138a8c2c9209d72d01a0
 
 [template-kumofs]
 filename = instance-kumofs.cfg.in
-md5sum = 69954583b5f87aec5ff8449dabaaee56
+md5sum = 45cc45510b59ceb730b6e38448b5c0c3
 
 [template-zope-conf]
 filename = zope.conf.in
@@ -78,7 +78,7 @@ md5sum = 3d8f3a440b7423c3b947c6ea4d775c6e
 
 [template-zeo]
 filename = instance-zeo.cfg.in
-md5sum = c4908c340dbd74f2d434d7e5deebff59
+md5sum = 0ba5735ab87ee53e2c203b1563b55ff0
 
 [template-zodb-base]
 filename = instance-zodb-base.cfg.in

stack/erp5/instance-kumofs.cfg.in

@@ -78,7 +78,7 @@ kumofs-data = ${:srv}/kumofs
 
 [resiliency-exclude-file]
 # Generate rdiff exclude file in case of resiliency
-recipe = slapos.recipe.template
+recipe = slapos.recipe.template:jinja2
 inline = {{ '{{ "**\\n" }}' }}
 output = ${directory:srv}/exporter.exclude
 

stack/erp5/instance-zeo.cfg.in

@@ -109,7 +109,7 @@ command = ${tidstorage:repozo-wrapper}
 # webrunner resiliency with erp5 inside.
 [{{ section("resiliency-exclude-file") }}]
 # Generate rdiff exclude file
-recipe = slapos.recipe.template
+recipe = slapos.recipe.template:jinja2
 inline = {{ '{{ "${directory:zodb}/**\\n${directory:log}/**\\n" }}' }}
 output = ${directory:srv}/exporter.exclude