Merge branch 'feature/kvm-follow-slapos-restart'

This work results with VM being restarted on parameter change.

The VM will be stopped in case of parameter change, then the system will try
to start it back. For external resources (like downloadable images) it will be
waited until such images are correctly present before starting back the VM.

Image download is done asynchronously with image-download-controller system.

software/kvm/buildout.hash.cfg

@@ -19,7 +19,7 @@ md5sum = 0d34ff81779115bf899f7bc752877b70
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2
-md5sum = 4f71b7616b9f4a2f968e89326a237768
+md5sum = bf0c01ac7493693bb57ebef00bb20fa0
 
 [template-kvm-cluster]
 filename = instance-kvm-cluster.cfg.jinja2.in
@@ -55,7 +55,7 @@ md5sum = b7e87479a289f472b634a046b44b5257
 
 [template-kvm-run]
 filename = template/template-kvm-run.in
-md5sum = adf05b4255269bc7d0eec479424405e4
+md5sum = b8cc7c76438212e0522ebede88649393
 
 [template-kvm-controller]
 filename = template/kvm-controller-run.in

software/kvm/instance-kvm-cluster-input-schema.json

@@ -359,7 +359,7 @@
             },
             "virtual-hard-drive-md5sum": {
               "title": "Checksum of virtual hard drive",
-              "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
+              "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
               "type": "string"
             },
             "virtual-hard-drive-gzipped": {

software/kvm/instance-kvm-input-schema.json

@@ -160,7 +160,7 @@
     },
     "virtual-hard-drive-md5sum": {
       "title": "Checksum of virtual hard drive",
-      "description": "MD5 checksum of virtual hard drive, used if virtual-hard-drive-url is specified.",
+      "description": "MD5 checksum of virtual hard drive, required if virtual-hard-drive-url is specified.",
       "type": "string"
     },
     "virtual-hard-drive-gzipped": {

software/kvm/instance-kvm.cfg.jinja2

@@ -17,6 +17,8 @@
 {% set nat_rule_list = slapparameter_dict.get('nat-rules', '22 80 443') -%}
 {% set disk_device_path = slapparameter_dict.get('disk-device-path', None) -%}
 {% set whitelist_domains = slapparameter_dict.get('whitelist-domains', '') -%}
+{% set virtual_hard_drive_url_enabled = 'virtual-hard-drive-url' in slapparameter_dict %}
+{% set virtual_hard_drive_url_gzipped = str(slapparameter_dict.get('virtual-hard-drive-gzipped', False)).lower() == 'true' %}
 {% set boot_image_url_list_enabled = 'boot-image-url-list' in slapparameter_dict %}
 {% set boot_image_url_select_enabled = 'boot-image-url-select' in slapparameter_dict %}
 {% set bootstrap_script_url = slapparameter_dict.get('bootstrap-script-url') -%}
@@ -56,6 +58,11 @@ public = ${:srv}/public/
 cron-entries = ${:etc}/cron.d
 crontabs = ${:etc}/crontabs
 cronstamps = ${:etc}/cronstamps
+{%- if virtual_hard_drive_url_enabled %}
+virtual-hard-drive-url-repository = ${:srv}/virtual-hard-drive-url-repository
+virtual-hard-drive-url-var = ${:var}/virtual-hard-drive-url
+virtual-hard-drive-url-expose = ${monitor-directory:private}/virtual-hard-drive-url
+{%- endif %}
 {%- if boot_image_url_list_enabled %}
 boot-image-url-list-repository = ${:srv}/boot-image-url-list-repository
 boot-image-url-list-var = ${:var}/boot-image-url-list
@@ -278,6 +285,106 @@ config-filename = ${boot-image-url-list-download-wrapper:error-state-file}
 ## boot-image-url-list support END
 {% endif %} {# if boot_image_url_list_enabled #}
 
+{% if virtual_hard_drive_url_enabled %}
+## virtual-hard-drive-url support BEGIN
+[empty-file-state-base-virtual-promise]
+<= monitor-promise-base
+module = check_file_state
+name = ${:_buildout_section_name_}.py
+config-state = empty
+# It's very hard to put the username and password correctly, after schema://
+# and before the host, as it's not the way how one can use monitor provided
+# information, so just show the information in the URL
+config-url = ${monitor-base:base-url}/private/virtual-hard-drive-url/${:filename} with username ${monitor-publish-parameters:monitor-user} and password ${monitor-publish-parameters:monitor-password}
+
+[virtual-hard-drive-url-source-config]
+recipe = slapos.recipe.template:jinja2
+template = inline:
+{%- raw %}
+  {{ virtual_hard_drive_url }}
+{% endraw -%}
+{# Enforce md5sum on virtual-hard-drive-url #}
+virtual-hard-drive-url = {{ slapparameter_dict['virtual-hard-drive-url'] }}#{{ slapparameter_dict['virtual-hard-drive-md5sum'] }}
+context =
+  key virtual_hard_drive_url :virtual-hard-drive-url
+rendered = ${directory:etc}/virtual-hard-drive-url.conf
+
+[virtual-hard-drive-url-processed-config]
+# compares if the current configuration has been used by
+# the virtual-hard-drive-url-download, if not, exposes it as not empty file with
+# information
+recipe = slapos.recipe.build
+install =
+  import os
+  import hashlib
+  if not os.path.exists(location):
+    os.mkdir(location)
+  with open('${:state-file}', 'w') as state_handler:
+    try:
+      with open('${:config-file}', 'rb') as config_handler, open('${:processed-md5sum}') as processed_handler:
+        config_md5sum = hashlib.md5(config_handler.read()).hexdigest()
+        processed_md5sum = processed_handler.read()
+        if config_md5sum == processed_md5sum:
+          state_handler.write('')
+        else:
+          state_handler.write('config %s != processed %s' % (config_md5sum, processed_md5sum))
+    except Exception as e:
+      state_handler.write(str(e))
+
+update = ${:install}
+config-file = ${virtual-hard-drive-url-source-config:rendered}
+state-filename = virtual-hard-drive-url-processed-config.state
+state-file = ${directory:virtual-hard-drive-url-expose}/${:state-filename}
+processed-md5sum = ${directory:virtual-hard-drive-url-var}/update-image-processed.md5sum
+
+[virtual-hard-drive-url-processed-config-promise]
+# promise to check if the configuration provided by the user has been already
+# processed by the virtual-hard-drive-url-download script, which runs asynchronously
+<= empty-file-state-base-virtual-promise
+filename = ${virtual-hard-drive-url-processed-config:state-filename}
+config-filename = ${virtual-hard-drive-url-processed-config:state-file}
+
+[virtual-hard-drive-url-json-config]
+# generates json configuration from user configuration
+recipe = plone.recipe.command
+command = {{ python_executable }} {{ image_download_config_creator }} ${virtual-hard-drive-url-source-config:rendered} ${:rendered} ${directory:virtual-hard-drive-url-repository} ${:error-state-file}
+update-command = ${:command}
+rendered = ${directory:virtual-hard-drive-url-var}/virtual-hard-drive-url.json
+error-state-filename = virtual-hard-drive-url-json-config-error.txt
+error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
+
+[virtual-hard-drive-url-config-state-promise]
+# promise to check if configuration has been parsed without errors
+<= empty-file-state-base-virtual-promise
+filename = ${virtual-hard-drive-url-json-config:error-state-filename}
+config-filename = ${virtual-hard-drive-url-json-config:error-state-file}
+
+[virtual-hard-drive-url-download-wrapper]
+# wrapper to execute virtual-hard-drive-url-download on each run
+recipe = slapos.cookbook:wrapper
+wrapper-path = ${directory:scripts}/virtual-hard-drive-url-updater
+command-line = {{ python_executable }} {{ image_download_controller }} ${virtual-hard-drive-url-json-config:rendered} {{ curl_executable_location }} ${:md5sum-state-file} ${:error-state-file} ${virtual-hard-drive-url-processed-config:processed-md5sum}
+md5sum-state-filename = virtual-hard-drive-url-download-controller-md5sum-fail.json
+md5sum-state-file = ${directory:virtual-hard-drive-url-expose}/${:md5sum-state-filename}
+error-state-filename = virtual-hard-drive-url-download-controller-error.text
+error-state-file = ${directory:virtual-hard-drive-url-expose}/${:error-state-filename}
+hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
+
+[virtual-hard-drive-url-download-md5sum-promise]
+# promise to report errors with problems with calculating md5sum of the
+# downloaded images
+<= empty-file-state-base-virtual-promise
+filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-filename}
+config-filename = ${virtual-hard-drive-url-download-wrapper:md5sum-state-file}
+
+[virtual-hard-drive-url-download-state-promise]
+# promise to report errors during download
+<= empty-file-state-base-virtual-promise
+filename = ${virtual-hard-drive-url-download-wrapper:error-state-filename}
+config-filename = ${virtual-hard-drive-url-download-wrapper:error-state-file}
+## virtual-hard-drive-url support END
+{% endif %} {# if virtual_hard_drive_url_enabled #}
+
 [kvm-controller-parameter-dict]
 python-path = {{ python_eggs_executable }}
 vnc-passwd = ${gen-passwd:passwd}
@@ -298,6 +405,11 @@ vnc-ip = ${:ipv4}
 vnc-port = 5901
 
 default-cdrom-iso = {{ debian_amd64_netinst_location }}
+{% if virtual_hard_drive_url_enabled %}
+virtual-hard-drive-url-json-config = ${virtual-hard-drive-url-json-config:rendered}
+{% else %}
+virtual-hard-drive-url-json-config =
+{% endif %}
 {% if boot_image_url_list_enabled %}
 boot-image-url-list-json-config = ${boot-image-url-list-json-config:rendered}
 {% else %}
@@ -425,14 +537,42 @@ ipv6-port = {{ external_port }}
 {%   endfor -%}
 {% endif -%}
 
+{%- set depend_section_list = [] %}
+{%- set hash_file_list = ['${kvm-run:rendered}'] %}
+{%- macro generate_depend_section(section, key) %}
+{%-  do depend_section_list.append('${' + section + ':command}' ) %}
+{%-  do hash_file_list.append('${' + key + '}') %}
+[{{ section }}]
+recipe = plone.recipe.command
+update-command = ${:command}
+command = [ ! -f {{ '${' + key + '}' }} ] && touch {{ '${' +  key + '}' }}
+{%- endmacro %}
+{#- Create depending sections, as state files appear late, so it's better to have empty file which will impact the hash anyway #}
+{%- if boot_image_url_list_enabled %}
+{{    generate_depend_section('boot-image-url-list-depend', 'boot-image-url-list-download-wrapper:md5sum-state-file') }}
+{%- endif %}
+{%- if boot_image_url_select_enabled %}
+{{    generate_depend_section('boot-image-url-select-depend', 'boot-image-url-select-download-wrapper:md5sum-state-file') }}
+{%- endif %}
+{%- if virtual_hard_drive_url_enabled %}
+{{    generate_depend_section('virtual-hard-drive-url-depend', 'virtual-hard-drive-url-download-wrapper:md5sum-state-file') }}
+{%- endif %}
+
 [kvm-instance]
+depends =
+{%- for depend_section in depend_section_list %}
+  {{ depend_section }}
+{%- endfor %}
 recipe = slapos.cookbook:wrapper
 socket-path = ${kvm-controller-parameter-dict:socket-path}
 wrapper-path = ${directory:services}/kvm
 command-line = ${kvm-run:rendered}
 kvm-controller = ${kvm-controller-wrapper:wrapper-path}
 hash-existing-files = ${buildout:directory}/software_release/buildout.cfg
-
+hash-files =
+{%- for hash_file in hash_file_list %}
+  {{ hash_file }}
+{%- endfor %}
 
 [kvm-controller-wrapper]
 recipe = slapos.cookbook:wrapper
@@ -1108,6 +1248,13 @@ parts =
   cron-service
   cron-entry-logrotate
   frontend-promise
+{% if virtual_hard_drive_url_enabled %}
+  virtual-hard-drive-url-download-wrapper
+  virtual-hard-drive-url-config-state-promise
+  virtual-hard-drive-url-download-md5sum-promise
+  virtual-hard-drive-url-download-state-promise
+  virtual-hard-drive-url-processed-config-promise
+{% endif %}
 {% if boot_image_url_list_enabled %}
   boot-image-url-list-download-wrapper
   boot-image-url-list-config-state-promise

software/kvm/template/template-kvm-run.in

@@ -6,10 +6,6 @@ import hashlib
 import os
 import socket
 import subprocess
-try:
-  from urllib.request import FancyURLopener
-except ImportError:
-  from urllib import FancyURLopener
 import gzip
 import shutil
 from random import shuffle
@@ -17,8 +13,6 @@ import glob
 import re
 import json
 
-import ssl
-
 # XXX: give all of this through parameter, don't use this as template, but as module
 qemu_img_path = {{ repr(parameter_dict["qemu-img-path"]) }}
 qemu_path = {{ repr(parameter_dict["qemu-path"]) }}
@@ -32,9 +26,6 @@ nbd_list = (('{{ parameter_dict.get("nbd-host") }}',
               {{ parameter_dict.get("nbd2-port") }}))
 default_cdrom_iso = '{{ parameter_dict.get("default-cdrom-iso") }}'
 
-virtual_hard_drive_url = '{{ parameter_dict.get("virtual-hard-drive-url") }}'.strip()
-virtual_hard_drive_md5sum = '{{ parameter_dict.get("virtual-hard-drive-md5sum") }}'.strip()
-virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
 nat_rules = '{{ parameter_dict.get("nat-rules") }}'.strip()
 use_tap = '{{ parameter_dict.get("use-tap") }}'.lower()
 use_nat = '{{ parameter_dict.get("use-nat") }}'.lower()
@@ -98,11 +89,8 @@ logfile = '{{ parameter_dict.get("log-file") }}'
 
 boot_image_url_list_json_config = '{{ parameter_dict.get("boot-image-url-list-json-config") }}'
 boot_image_url_select_json_config = '{{ parameter_dict.get("boot-image-url-select-json-config") }}'
-
-if hasattr(ssl, '_create_unverified_context') and url_check_certificate == 'false':
-  opener = FancyURLopener(context=ssl._create_unverified_context())
-else:
-  opener = FancyURLopener({})
+virtual_hard_drive_url_json_config = '{{ parameter_dict.get("virtual-hard-drive-url-json-config") }}'
+virtual_hard_drive_gzipped = '{{ parameter_dict.get("virtual-hard-drive-gzipped") }}'.strip().lower()
 
 def md5Checksum(file_path):
     with open(file_path, 'rb') as fh:
@@ -163,39 +151,34 @@ def getMapStorageList(disk_storage_dict, external_disk_number):
       lf.write('%s' % external_disk_number)
   return id_list, external_disk_number
 
-# Download existing hard drive if needed at first boot
-if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url != '':
-  print('Downloading virtual hard drive...')
-  try:
-    downloaded_disk = disk_info_list[0]['path']
+# Use downloaded virtual-hard-drive-url
+if len(disk_info_list) == 1 and not os.path.exists(disk_info_list[0]['path']) and virtual_hard_drive_url_json_config != '':
+  print('Using virtual hard drive...')
+  with open(virtual_hard_drive_url_json_config) as fh:
+    image_config = json.load(fh)
+  if image_config['error-amount'] == 0:
+    image = image_config['image-list'][0]
+    downloaded_image = os.path.join(image_config['destination-directory'], image['destination'])
+    # previous version was using disk in place, but here it would result with
+    # redownload, so copy it
     if virtual_hard_drive_gzipped == 'true':
-      downloaded_disk = '%s.gz' % disk_info_list[0]['path']
-    opener.retrieve(virtual_hard_drive_url, downloaded_disk)
-  except:
-    if os.path.exists(downloaded_disk):
-      os.remove(downloaded_disk)
-    raise
-  md5sum = virtual_hard_drive_md5sum.strip()
-  if md5sum:
-    print('Checking MD5 checksum...')
-    local_md5sum = md5Checksum(downloaded_disk)
-    if local_md5sum != md5sum:
-      os.remove(downloaded_disk)
-      raise Exception('MD5 mismatch. MD5 of local file is %s, Specified MD5 is %s.' % (
-          local_md5sum, md5sum))
-    print('MD5sum check passed.')
+      try:
+        with open(disk_info_list[0]['path'], 'wb') as d_fh:
+          with gzip.open(downloaded_image, 'rb') as s_fh:
+            shutil.copyfileobj(s_fh, d_fh)
+      except Exception:
+        if os.path.exists(disk_info_list[0]['path']):
+          os.unlink(disk_info_list[0]['path'])
+        raise
+    else:
+      try:
+        shutil.copyfile(downloaded_image, disk_info_list[0]['path'])
+      except Exception:
+        if os.path.exists(disk_info_list[0]['path']):
+          os.unlink(disk_info_list[0]['path'])
+        raise
   else:
-    print('Warning: not checksum specified.')
-  if downloaded_disk.endswith('.gz'):
-    try:
-      with open(disk_info_list[0]['path'], 'w') as disk:
-        with gzip.open(downloaded_disk, 'rb') as disk_gz:
-          shutil.copyfileobj(disk_gz, disk)
-    except Exception:
-      if os.path.exists(disk_info_list[0]['path']):
-        os.remove(disk_info_list[0]['path'])
-      raise
-    os.remove(downloaded_disk)
+    raise ValueError('virtual-hard-drive-url not ready yet')
 
 # Create disk if doesn't exist
 # XXX: move to Buildout profile
@@ -378,6 +361,9 @@ else:
             '-drive',
             'file=%s,media=cdrom' % (link,)
           ])
+    else:
+      raise ValueError('boot-image-url-select not ready yet')
+
   if boot_image_url_list_json_config:
     # Support boot-image-url-list
     with open(boot_image_url_list_json_config) as fh:
@@ -390,6 +376,8 @@ else:
             '-drive',
             'file=%s,media=cdrom' % (link,)
           ])
+    else:
+      raise ValueError('boot-image-url-list not ready yet')
   # Always add by default the default image
   kvm_argument_list.extend([
       '-drive', 'file=%s,media=cdrom' % default_cdrom_iso

software/kvm/test/test.py

@@ -28,6 +28,7 @@
 import six.moves.http_client as httplib
 import json
 import os
+import glob
 import hashlib
 import psutil
 import requests
@@ -43,6 +44,7 @@ from six.moves import SimpleHTTPServer
 import multiprocessing
 import time
 import shutil
+import sys
 
 from slapos.recipe.librecipe import generateHashFromFiles
 from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
@@ -120,12 +122,25 @@ class KvmMixin(object):
         'software_release/buildout.cfg',
       ]
     ])
+    # find bin/kvm_raw
+    kvm_raw_list = glob.glob(
+      os.path.join(self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
+    self.assertEqual(1, len(kvm_raw_list))  # allow to work only with one
+    hash_file_list = [
+      kvm_raw_list[0],
+      'software_release/buildout.cfg',
+    ]
+    kvm_hash_value = generateHashFromFiles([
+      os.path.join(self.computer_partition_root_path, hash_file)
+      for hash_file in hash_file_list
+    ])
     with self.slap.instance_supervisor_rpc as supervisor:
       running_process_info = '\n'.join(sorted([
         '%(group)s:%(name)s %(statename)s' % q for q
         in supervisor.getAllProcessInfo()
         if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
-    return running_process_info.replace(hash_value, '{hash}')
+    return running_process_info.replace(
+      hash_value, '{hash}').replace(kvm_hash_value, '{kvm-hash-value}')
 
   def raising_waitForInstance(self, max_retry):
     with self.assertRaises(SlapOSNodeCommandError):
@@ -176,7 +191,7 @@ i0:6tunnel-10443-{hash}-on-watch RUNNING
 i0:bootstrap-monitor EXITED
 i0:certificate_authority-{hash}-on-watch RUNNING
 i0:crond-{hash}-on-watch RUNNING
-i0:kvm-{hash}-on-watch RUNNING
+i0:kvm-{kvm-hash-value}-on-watch RUNNING
 i0:kvm_controller EXITED
 i0:monitor-httpd-{hash}-on-watch RUNNING
 i0:monitor-httpd-graceful EXITED
@@ -416,10 +431,11 @@ class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
           "test-machine2": dict(bootstrap_machine_param_dict, **{
               # Debian 9 image
               "virtual-hard-drive-url":
-              "http://shacache.org/shacache/ce07873dbab7fa8501d1bf5565c2737b2"
-              "eed6c8b9361b4997b21daf5f5d1590972db9ac00131cc5b27d9aa353f2f940"
-              "71e073f9980cc61badd6d2427f592e6e8",
-              "virtual-hard-drive-md5sum": "2b113e3cd8276b9740189622603d6f99"
+              "http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
+              "57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
+              "a508930046a78cffea6e0e29d03345333cc",
+              "virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
+              "virtual-hard-drive-gzipped": True,
           })
       }
     }))}
@@ -499,7 +515,7 @@ ir2:bootstrap-monitor EXITED
 ir2:certificate_authority-{hash}-on-watch RUNNING
 ir2:crond-{hash}-on-watch RUNNING
 ir2:equeue-on-watch RUNNING
-ir2:kvm-{hash}-on-watch RUNNING
+ir2:kvm-{kvm-hash-value}-on-watch RUNNING
 ir2:kvm_controller EXITED
 ir2:monitor-httpd-{hash}-on-watch RUNNING
 ir2:monitor-httpd-graceful EXITED
@@ -1282,7 +1298,7 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
       'disk-device-path': '/dev/virt0 /dev/virt1',
       'wipe-disk-ondestroy': True
     })
-    self.slap.waitForInstance(max_retry=2)
+    self.raising_waitForInstance(3)
     instance_path = os.path.join(
       self.slap.instance_directory, self.kvm_instance_partition_reference)
 
@@ -1298,3 +1314,210 @@ class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
 dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
       )
     self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))
+
+
+@skipUnlessKvm
+class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
+  __partition_reference__ = 'idc'
+  maxDiff = None
+
+  def setUp(self):
+    super(TestImageDownloadController, self).setUp()
+    self.working_directory = tempfile.mkdtemp()
+    self.destination_directory = os.path.join(
+      self.working_directory, 'destination')
+    os.mkdir(self.destination_directory)
+    self.config_json = os.path.join(
+      self.working_directory, 'config.json')
+    self.md5sum_fail_file = os.path.join(
+      self.working_directory, 'md5sum_fail_file')
+    self.error_state_file = os.path.join(
+      self.working_directory, 'error_state_file')
+    self.processed_md5sum = os.path.join(
+      self.working_directory, 'processed_md5sum')
+    self.startImageHttpServer()
+    self.image_download_controller = os.path.join(
+      self.slap.instance_directory, self.__partition_reference__ + '0',
+      'software_release', 'parts', 'image-download-controller',
+      'image-download-controller')
+
+  def tearDown(self):
+    self.stopImageHttpServer()
+    shutil.rmtree(self.working_directory)
+    super(InstanceTestCase, self).tearDown()
+
+  def callImageDownloadController(self, *args):
+    call_list = [sys.executable, self.image_download_controller] + list(args)
+    try:
+      return (0, subprocess.check_output(
+        call_list, stderr=subprocess.STDOUT).decode('utf-8'))
+    except subprocess.CalledProcessError as e:
+      return (e.returncode, e.output.decode('utf-8'))
+
+  def runImageDownloadControlerWithDict(self, json_dict):
+    with open(self.config_json, 'w') as fh:
+      json.dump(json_dict, fh, indent=2)
+    return self.callImageDownloadController(
+      self.config_json,
+      'curl',  # comes from test environemnt, considered to be recent enough
+      self.md5sum_fail_file,
+      self.error_state_file,
+      self.processed_md5sum
+    )
+
+  def assertFileContent(self, path, content):
+    self.assertTrue(os.path.exists, path)
+    with open(path, 'r') as fh:
+      self.assertEqual(
+        fh.read(),
+        content)
+
+  def test(self):
+    json_dict = {
+      'error-amount': 0,
+      'config-md5sum': 'config-md5sum',
+      'destination-directory': self.destination_directory,
+      'image-list': [
+        {
+          'destination-tmp': 'tmp',
+          'url': self.fake_image,
+          'destination': 'destination',
+          'link': 'image_001',
+          'gzipped': False,
+          'md5sum': self.fake_image_md5sum,
+        }
+      ]
+    }
+    code, result = self.runImageDownloadControlerWithDict(
+      json_dict
+    )
+    self.assertEqual(
+      (code, result.strip()),
+      (0, """
+INF: Storing errors in %(error_state_file)s
+INF: %(fake_image)s : Downloading
+INF: %(fake_image)s : Stored with checksum %(checksum)s
+INF: %(fake_image)s : Symlinking %(symlink)s -> %(destination)s
+""".strip() % {
+        'fake_image': self.fake_image,
+        'checksum': self.fake_image_md5sum,
+        'error_state_file': self.error_state_file,
+        'symlink': os.path.join(self.destination_directory, 'image_001'),
+        'destination': os.path.join(self.destination_directory, 'destination'),
+      })
+    )
+    self.assertFileContent(self.md5sum_fail_file, '')
+    self.assertFileContent(self.error_state_file, '')
+    self.assertFileContent(self.processed_md5sum, 'config-md5sum')
+    self.assertFalse(
+      os.path.exists(os.path.join(self.destination_directory, 'tmp')))
+    self.assertFileContent(
+      os.path.join(self.destination_directory, 'destination'),
+      'fake_image_content'
+    )
+
+    # Nothing happens if all is downloaded
+    code, result = self.runImageDownloadControlerWithDict(
+      json_dict
+    )
+    self.assertEqual(
+      (code, result.strip()),
+      (0, """
+INF: Storing errors in %(error_state_file)s
+INF: %(fake_image)s : already downloaded
+""".strip() % {
+        'fake_image': self.fake_image,
+        'checksum': self.fake_image_md5sum,
+        'error_state_file': self.error_state_file,
+        'symlink': os.path.join(self.destination_directory, 'image_001'),
+        'destination': os.path.join(self.destination_directory, 'destination'),
+      })
+    )
+
+  def test_fail(self):
+    json_dict = {
+      'error-amount': 0,
+      'config-md5sum': 'config-md5sum',
+      'destination-directory': self.destination_directory,
+      'image-list': [
+        {
+          'destination-tmp': 'tmp',
+          'url': self.fake_image,
+          'destination': 'destination',
+          'link': 'image_001',
+          'gzipped': False,
+          'md5sum': self.fake_image_wrong_md5sum,
+        }
+      ]
+    }
+    for try_num in range(1, 5):
+      code, result = self.runImageDownloadControlerWithDict(
+        json_dict
+      )
+      self.assertEqual(
+        (code, result.strip()),
+        (1, """
+INF: Storing errors in %(error_state_file)s
+INF: %(fake_image)s : Downloading
+""".  strip() % {
+          'fake_image': self.fake_image,
+          'error_state_file': self.error_state_file,
+          'symlink': os.path.join(self.destination_directory, 'image_001'),
+          'destination': os.path.join(
+            self.destination_directory, 'destination'),
+        })
+      )
+      fake_image_url = '#'.join([
+        self.fake_image, self.fake_image_wrong_md5sum])
+      self.assertFileContent(
+        self.md5sum_fail_file, """{
+  "%s": %s
+}""" % (fake_image_url, try_num))
+      self.assertFileContent(
+        self.error_state_file, """
+        ERR: %(fake_image)s : MD5 mismatch expected is %(wrong_checksum)s """
+        """but got instead %(real_checksum)s""".strip() % {
+          'fake_image': self.fake_image,
+          'wrong_checksum': self.fake_image_wrong_md5sum,
+          'real_checksum': self.fake_image_md5sum,
+        })
+      self.assertFileContent(self.processed_md5sum, 'config-md5sum')
+      self.assertFalse(
+        os.path.exists(os.path.join(self.destination_directory, 'tmp')))
+      self.assertFalse(
+        os.path.exists(
+          os.path.join(self.destination_directory, 'destination')))
+
+    code, result = self.runImageDownloadControlerWithDict(
+      json_dict
+    )
+    self.assertEqual(
+      (code, result.strip()),
+      (1, """
+INF: Storing errors in %(error_state_file)s
+""".  strip() % {
+        'fake_image': self.fake_image,
+        'error_state_file': self.error_state_file,
+        'symlink': os.path.join(self.destination_directory, 'image_001'),
+        'destination': os.path.join(
+          self.destination_directory, 'destination'),
+      })
+    )
+    fake_image_url = '#'.join([
+      self.fake_image, self.fake_image_wrong_md5sum])
+    self.assertFileContent(
+      self.md5sum_fail_file, """{
+  "%s": %s
+}""" % (fake_image_url, 4))
+    self.assertFileContent(
+      self.error_state_file, """
+      ERR: %(fake_image)s : Checksum is incorrect after 4 tries, will not """
+      """retry""".strip() % {
+        'fake_image': self.fake_image,
+      })
+    self.assertFileContent(self.processed_md5sum, 'config-md5sum')
+    self.assertFalse(
+      os.path.exists(os.path.join(self.destination_directory, 'tmp')))
+    self.assertFalse(
+      os.path.exists(
+        os.path.join(self.destination_directory, 'destination')))