##############################################################################
#
# Copyright (c) 2018 Nexedi SA and Contributors. All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 3
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
#
##############################################################################

import six.moves.http_client as httplib
import json
import os
import glob
import hashlib
import psutil
import requests
import six
import slapos.util
import sqlite3
from six.moves.urllib.parse import parse_qs, urlparse
import unittest
import subprocess
import tempfile
import six.moves.socketserver as SocketServer
from six.moves import SimpleHTTPServer
import multiprocessing
import time
import shutil
import sys

from slapos.recipe.librecipe import generateHashFromFiles
from slapos.testing.testcase import makeModuleSetUpAndTestCaseClass
from slapos.slap.standalone import SlapOSNodeCommandError
from slapos.testing.utils import findFreeTCPPort

has_kvm = os.access('/dev/kvm', os.R_OK | os.W_OK)
skipUnlessKvm = unittest.skipUnless(has_kvm, 'kvm not loaded or not allowed')

if has_kvm:
  setUpModule, InstanceTestCase = makeModuleSetUpAndTestCaseClass(
    os.path.abspath(
      os.path.join(os.path.dirname(__file__), '..',
                   'software%s.cfg' % ("-py3" if six.PY3 else ""))))
else:
  setUpModule, InstanceTestCase = None, unittest.TestCase

  class SanityCheckTestCase(unittest.TestCase):
    def test_kvm_sanity_check(self):
      self.fail('This environment is not usable for kvm testing,'
                ' as it lacks kvm_intel kernel module')

bootstrap_common_param_dict = {
    # the bootstrap script is vm-bootstrap
    "bootstrap-script-url":
    "http://shacache.org/shacache/05105cd25d1ad798b71fd46a206c9b73da2c285a078"
    "af33d0e739525a595886785725a68811578bc21f75d0a97700a66d5e75bce5b2721ca455"
    "6a0734cb13e65#c98825aa1b6c8087914d2bfcafec3058",
    "slave-frontend": {
        "slave-frontend-dict": {}
    },
    "authorized-keys": [
        "ssh-rsa %s key_one" % ("A" * 372),
        "ssh-rsa %s key_two" % ("B" * 372),
        "ssh-rsa %s key_three" % ("C" * 372)
    ],
    "fw-restricted-access": "off",
    "fw-authorized-sources": [],
    "fw-reject-sources": ["10.32.0.0/13"]
}

bootstrap_machine_param_dict = {
    "computer-guid": "local",
    "disable-ansible-promise": True,
    "state": "started",
    "auto-ballooning": True,
    "ram-size": 4096,
    "cpu-count": 2,
    "disk-size": 50,
    # Debian 10 image
    "virtual-hard-drive-url":
    "http://shacache.org/shacache/9d3e6d017754fdd08e5ecf78093dec27fd792fb183d"
    "f6146006adf003b6f4b98c0388d5a11566627101f7855d77f60e3dd4ba7ce66850f4a8f0"
    "30573b904d5ab",
    "virtual-hard-drive-md5sum": "b7928d7b0a2b5e2888f5ddf68f5fe422",
    "virtual-hard-drive-gzipped": False,
    "hard-drive-url-check-certificate": False,
    "use-tap": True,
    "use-nat": True,
    "nat-restrict-mode": True,
    "enable-vhost": True,
    "external-disk-number": 1,
    "external-disk-size": 100,
    "external-disk-format": "qcow2",
    "enable-monitor": True,
    "keyboard-layout-language": "fr"
}


class KvmMixin(object):
  def getProcessInfo(self):
    hash_value = generateHashFromFiles([
      os.path.join(self.computer_partition_root_path, hash_file)
      for hash_file in [
        'software_release/buildout.cfg',
      ]
    ])
    # find bin/kvm_raw
    kvm_raw_list = glob.glob(
      os.path.join(self.slap.instance_directory, '*', 'bin', 'kvm_raw'))
    self.assertEqual(1, len(kvm_raw_list))  # allow to work only with one
    hash_file_list = [
      kvm_raw_list[0],
      'software_release/buildout.cfg',
    ]
    kvm_hash_value = generateHashFromFiles([
      os.path.join(self.computer_partition_root_path, hash_file)
      for hash_file in hash_file_list
    ])
    with self.slap.instance_supervisor_rpc as supervisor:
      running_process_info = '\n'.join(sorted([
        '%(group)s:%(name)s %(statename)s' % q for q
        in supervisor.getAllProcessInfo()
        if q['name'] != 'watchdog' and q['group'] != 'watchdog']))
    return running_process_info.replace(
      hash_value, '{hash}').replace(kvm_hash_value, '{kvm-hash-value}')

  def raising_waitForInstance(self, max_retry):
    with self.assertRaises(SlapOSNodeCommandError):
      self.slap.waitForInstance(max_retry=max_retry)

  def rerequestInstance(self, parameter_dict, state='started'):
    software_url = self.getSoftwareURL()
    software_type = self.getInstanceSoftwareType()
    return self.slap.request(
        software_release=software_url,
        software_type=software_type,
        partition_reference=self.default_partition_reference,
        partition_parameter_kw=parameter_dict,
        state=state)


@skipUnlessKvm
class TestInstance(InstanceTestCase, KvmMixin):
  __partition_reference__ = 'i'

  def test(self):
    connection_parameter_dict = self\
      .computer_partition.getConnectionParameterDict()
    present_key_list = []
    assert_key_list = [
     'backend-url', 'url', 'monitor-setup-url', 'ipv6-network-info',
     'tap-ipv4', 'tap-ipv6']
    for k in assert_key_list:
      if k in connection_parameter_dict:
        present_key_list.append(k)
        connection_parameter_dict.pop(k)
    self.assertEqual(
      connection_parameter_dict,
      {
        'ipv6': self._ipv6_address,
        'maximum-extra-disk-amount': '0',
        'monitor-base-url': 'https://[%s]:8026' % (self._ipv6_address,),
        'nat-rule-port-tcp-22': '%s : 10022' % (self._ipv6_address,),
        'nat-rule-port-tcp-443': '%s : 10443' % (self._ipv6_address,),
        'nat-rule-port-tcp-80': '%s : 10080' % (self._ipv6_address,),
      }
    )
    self.assertEqual(set(present_key_list), set(assert_key_list))
    self.assertEqual(
      """i0:6tunnel-10022-{hash}-on-watch RUNNING
i0:6tunnel-10080-{hash}-on-watch RUNNING
i0:6tunnel-10443-{hash}-on-watch RUNNING
i0:bootstrap-monitor EXITED
i0:certificate_authority-{hash}-on-watch RUNNING
i0:crond-{hash}-on-watch RUNNING
i0:kvm-{kvm-hash-value}-on-watch RUNNING
i0:kvm_controller EXITED
i0:monitor-httpd-{hash}-on-watch RUNNING
i0:monitor-httpd-graceful EXITED
i0:websockify-{hash}-on-watch RUNNING
i0:whitelist-domains-download-{hash} RUNNING
i0:whitelist-firewall-{hash} RUNNING""",
      self.getProcessInfo()
    )


class MonitorAccessMixin(object):
  def sqlite3_connect(self):
    sqlitedb_file = os.path.join(
      os.path.abspath(
        os.path.join(
          self.slap.instance_directory, os.pardir
        )
      ), 'var', 'proxy.db'
    )
    return sqlite3.connect(sqlitedb_file)

  def get_all_instantiated_partition_list(self):
    connection = self.sqlite3_connect()

    def dict_factory(cursor, row):
      d = {}
      for idx, col in enumerate(cursor.description):
        d[col[0]] = row[idx]
      return d
    connection.row_factory = dict_factory
    cursor = connection.cursor()

    cursor.execute(
      "SELECT reference, xml, connection_xml, partition_reference, "
      "software_release, requested_state, software_type "
      "FROM partition14 "
      "WHERE slap_state='busy'")
    return cursor.fetchall()

  def test_access_monitor(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    monitor_setup_url = connection_parameter_dict['monitor-setup-url']
    monitor_url_with_auth = 'https' + monitor_setup_url.split('https')[2]

    auth = parse_qs(urlparse(monitor_url_with_auth).path)

    # check that monitor-base-url for all partitions in the tree are accessible
    # with published username and password
    partition_with_monitor_base_url_count = 0
    for partition_information in self.get_all_instantiated_partition_list():
      connection_xml = partition_information.get('connection_xml')
      if not connection_xml:
        continue
      connection_dict = slapos.util.xml2dict(
        connection_xml if six.PY3 else connection_xml.encode('utf-8'))
      monitor_base_url = connection_dict.get('monitor-base-url')
      if not monitor_base_url:
        continue
      result = requests.get(
        monitor_base_url, verify=False, auth=(
          auth['username'][0],
          auth['password'][0])
      )

      self.assertEqual(
        httplib.OK,
        result.status_code
      )
      partition_with_monitor_base_url_count += 1
    self.assertEqual(
      self.expected_partition_with_monitor_base_url_count,
      partition_with_monitor_base_url_count
    )


@skipUnlessKvm
class TestAccessDefault(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'ad'
  expected_partition_with_monitor_base_url_count = 1

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(connection_parameter_dict['url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)
    self.assertNotIn('url-additional', connection_parameter_dict)


@skipUnlessKvm
class TestAccessDefaultAdditional(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'ada'
  expected_partition_with_monitor_base_url_count = 1

  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'frontend-additional-instance-guid': 'SOMETHING'
    }

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()

    result = requests.get(connection_parameter_dict['url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)

    result = requests.get(
      connection_parameter_dict['url-additional'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


@skipUnlessKvm
class TestAccessDefaultBootstrap(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'adb'
  expected_partition_with_monitor_base_url_count = 1
  # as few gigabytes are being downloaded, wait a bit longer
  instance_max_retry = 100

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps(dict(
      bootstrap_common_param_dict, **bootstrap_machine_param_dict))}

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()

    result = requests.get(connection_parameter_dict['url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


@skipUnlessKvm
class TestAccessKvmCluster(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'akc'
  expected_partition_with_monitor_base_url_count = 2

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True
        }
      }
    })}

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)
    self.assertNotIn('KVM0-url-additional', connection_parameter_dict)


@skipUnlessKvm
class TestAccessKvmClusterAdditional(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'akca'
  expected_partition_with_monitor_base_url_count = 2

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "frontend": {
        'frontend-additional-instance-guid': 'SOMETHING',
      },
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True,
        }
      }
    })}

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(connection_parameter_dict['KVM0-url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)

    result = requests.get(
      connection_parameter_dict['KVM0-url-additional'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


@skipUnlessKvm
class TestAccessKvmClusterBootstrap(MonitorAccessMixin, InstanceTestCase):
  __partition_reference__ = 'akcb'
  expected_partition_with_monitor_base_url_count = 3
  # as few gigabytes are being downloaded, wait a bit longer
  instance_max_retry = 100

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps(dict(bootstrap_common_param_dict, **{
      "kvm-partition-dict": {
          "test-machine1": bootstrap_machine_param_dict,
          "test-machine2": dict(bootstrap_machine_param_dict, **{
              # Debian 9 image
              "virtual-hard-drive-url":
              "http://shacache.org/shacache/93aeb72a556fe88d9889ce16558dfead"
              "57a3c8f0a80d0e04ebdcd4a5830dfa6403e3976cc896b8332e74f202fccbd"
              "a508930046a78cffea6e0e29d03345333cc",
              "virtual-hard-drive-md5sum": "cdca79619ba987c40b98a8e31d281e4a",
              "virtual-hard-drive-gzipped": True,
          })
      }
    }))}

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(
      connection_parameter_dict['test-machine1-url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)
    result = requests.get(
      connection_parameter_dict['test-machine2-url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


@skipUnlessKvm
class TestInstanceResilient(InstanceTestCase, KvmMixin):
  __partition_reference__ = 'ir'
  instance_max_retry = 20

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'

  def test(self):
    connection_parameter_dict = self\
      .computer_partition.getConnectionParameterDict()
    present_key_list = []
    assert_key_list = [
     'monitor-password', 'takeover-kvm-1-password', 'backend-url', 'url',
     'monitor-setup-url', 'ipv6-network-info']
    for k in assert_key_list:
      if k in connection_parameter_dict:
        present_key_list.append(k)
        connection_parameter_dict.pop(k)
    self.assertEqual(
      connection_parameter_dict,
      {
        'feed-url-kvm-1-pull': 'http://[%s]:8088/get/local-ir0-kvm-1-pull' % (
          self._ipv6_address,),
        'feed-url-kvm-1-push': 'http://[%s]:8088/get/local-ir0-kvm-1-push' % (
          self._ipv6_address,),
        'ipv6': self._ipv6_address,
        'monitor-base-url': 'https://[%s]:8160' % (self._ipv6_address,),
        'monitor-user': 'admin',
        'takeover-kvm-1-url': 'http://[%s]:9263/' % (self._ipv6_address,),
      }
    )
    self.assertEqual(set(present_key_list), set(assert_key_list))

    self.assertEqual(
      """ir0:bootstrap-monitor EXITED
ir0:certificate_authority-{hash}-on-watch RUNNING
ir0:crond-{hash}-on-watch RUNNING
ir0:monitor-httpd-{hash}-on-watch RUNNING
ir0:monitor-httpd-graceful EXITED
ir1:bootstrap-monitor EXITED
ir1:certificate_authority-{hash}-on-watch RUNNING
ir1:crond-{hash}-on-watch RUNNING
ir1:equeue-on-watch RUNNING
ir1:monitor-httpd-{hash}-on-watch RUNNING
ir1:monitor-httpd-graceful EXITED
ir1:notifier-on-watch RUNNING
ir1:pbs_sshkeys_authority-on-watch RUNNING
ir2:6tunnel-10022-{hash}-on-watch RUNNING
ir2:6tunnel-10080-{hash}-on-watch RUNNING
ir2:6tunnel-10443-{hash}-on-watch RUNNING
ir2:bootstrap-monitor EXITED
ir2:certificate_authority-{hash}-on-watch RUNNING
ir2:crond-{hash}-on-watch RUNNING
ir2:equeue-on-watch RUNNING
ir2:kvm-{kvm-hash-value}-on-watch RUNNING
ir2:kvm_controller EXITED
ir2:monitor-httpd-{hash}-on-watch RUNNING
ir2:monitor-httpd-graceful EXITED
ir2:notifier-on-watch RUNNING
ir2:resilient_sshkeys_authority-on-watch RUNNING
ir2:sshd-graceful EXITED
ir2:sshd-on-watch RUNNING
ir2:websockify-{hash}-on-watch RUNNING
ir2:whitelist-domains-download-{hash} RUNNING
ir2:whitelist-firewall-{hash} RUNNING
ir3:bootstrap-monitor EXITED
ir3:certificate_authority-{hash}-on-watch RUNNING
ir3:crond-{hash}-on-watch RUNNING
ir3:equeue-on-watch RUNNING
ir3:monitor-httpd-{hash}-on-watch RUNNING
ir3:monitor-httpd-graceful EXITED
ir3:notifier-on-watch RUNNING
ir3:resilient-web-takeover-httpd-on-watch RUNNING
ir3:resilient_sshkeys_authority-on-watch RUNNING
ir3:sshd-graceful EXITED
ir3:sshd-on-watch RUNNING""",
      self.getProcessInfo()
    )


@skipUnlessKvm
class TestAccessResilientAdditional(InstanceTestCase):
  __partition_reference__ = 'ara'
  expected_partition_with_monitor_base_url_count = 1

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'

  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'frontend-additional-instance-guid': 'SOMETHING'
    }

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()

    result = requests.get(connection_parameter_dict['url'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)

    result = requests.get(
      connection_parameter_dict['url-additional'], verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>noVNC</title>', result.text)


class TestInstanceNbdServer(InstanceTestCase):
  __partition_reference__ = 'ins'
  instance_max_retry = 5

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'nbd'

  @classmethod
  def getInstanceParameterDict(cls):
    # port 8080 is used by testnode, use another one
    return {
      'otu-port': '8090'
    }

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()
    result = requests.get(
      connection_parameter_dict['upload_url'].strip(), verify=False)
    self.assertEqual(
      httplib.OK,
      result.status_code
    )
    self.assertIn('<title>Upload new File</title>', result.text)
    self.assertIn("WARNING", connection_parameter_dict['status_message'])


class FakeImageHandler(SimpleHTTPServer.SimpleHTTPRequestHandler):
  def log_message(self, *args):
    if os.environ.get('SLAPOS_TEST_DEBUG'):
      return SimpleHTTPServer.SimpleHTTPRequestHandler.log_message(self, *args)
    else:
      return


class FakeImageServerMixin(KvmMixin):
  def startImageHttpServer(self):
    self.image_source_directory = tempfile.mkdtemp()
    server = SocketServer.TCPServer(
      (self._ipv4_address, findFreeTCPPort(self._ipv4_address)),
      FakeImageHandler)

    fake_image_content = b'fake_image_content'
    self.fake_image_md5sum = hashlib.md5(fake_image_content).hexdigest()
    with open(os.path.join(
      self.image_source_directory, self.fake_image_md5sum), 'wb') as fh:
      fh.write(fake_image_content)

    fake_image2_content = b'fake_image2_content'
    self.fake_image2_md5sum = hashlib.md5(fake_image2_content).hexdigest()
    with open(os.path.join(
      self.image_source_directory, self.fake_image2_md5sum), 'wb') as fh:
      fh.write(fake_image2_content)

    self.fake_image_wrong_md5sum = self.fake_image2_md5sum

    url = 'http://%s:%s' % server.server_address
    self.fake_image = '/'.join([url, self.fake_image_md5sum])
    self.fake_image2 = '/'.join([url, self.fake_image2_md5sum])

    old_dir = os.path.realpath(os.curdir)
    os.chdir(self.image_source_directory)
    try:
      self.server_process = multiprocessing.Process(
        target=server.serve_forever, name='FakeImageHttpServer')
      self.server_process.start()
    finally:
      os.chdir(old_dir)

  def stopImageHttpServer(self):
    self.logger.debug('Stopping process %s' % (self.server_process,))
    self.server_process.join(10)
    self.server_process.terminate()
    time.sleep(0.1)
    if self.server_process.is_alive():
      self.logger.warning(
        'Process %s still alive' % (self.server_process, ))

    shutil.rmtree(self.image_source_directory)


@skipUnlessKvm
class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
  __partition_reference__ = 'biul'
  kvm_instance_partition_reference = 'biul0'

  # variations
  key = 'boot-image-url-list'
  test_input = "%s#%s\n%s#%s"
  image_directory = 'boot-image-url-list-repository'
  config_state_promise = 'boot-image-url-list-config-state-promise.py'
  download_md5sum_promise = 'boot-image-url-list-download-md5sum-promise.py'
  download_state_promise = 'boot-image-url-list-download-state-promise.py'

  bad_value = "jsutbad"
  incorrect_md5sum_value_image = "%s#"
  incorrect_md5sum_value = "url#asdasd"
  single_image_value = "%s#%s"
  unreachable_host_value = "evennotahost#%s"
  too_many_image_value = """
      image1#11111111111111111111111111111111
      image2#22222222222222222222222222222222
      image3#33333333333333333333333333333333
      image4#44444444444444444444444444444444
      image5#55555555555555555555555555555555
      image6#66666666666666666666666666666666
      """

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'default'

  @classmethod
  def getInstanceParameterDict(cls):
    # start with empty, but working configuration
    return {}

  def setUp(self):
    super(InstanceTestCase, self).setUp()
    self.startImageHttpServer()

  def tearDown(self):
    # clean up the instance for other tests
    # 1st remove all images...
    self.rerequestInstance({self.key: ''})
    self.slap.waitForInstance(max_retry=10)
    # 2nd ...move instance to "default" state
    self.rerequestInstance({})
    self.slap.waitForInstance(max_retry=10)
    self.stopImageHttpServer()
    super(InstanceTestCase, self).tearDown()

  def test(self):
    partition_parameter_kw = {
      self.key: self.test_input % (
        self.fake_image, self.fake_image_md5sum, self.fake_image2,
        self.fake_image2_md5sum)
    }
    self.rerequestInstance(partition_parameter_kw)
    self.slap.waitForInstance(max_retry=10)
    # check that image is correctly downloaded and linked
    kvm_instance_partition = os.path.join(
      self.slap.instance_directory, self.kvm_instance_partition_reference)
    image_repository = os.path.join(
      kvm_instance_partition, 'srv', self.image_directory)
    image = os.path.join(image_repository, self.fake_image_md5sum)
    image_link = os.path.join(image_repository, 'image_001')
    self.assertTrue(os.path.exists(image))
    with open(image, 'rb') as fh:
      image_md5sum = hashlib.md5(fh.read()).hexdigest()
    self.assertEqual(image_md5sum, self.fake_image_md5sum)
    self.assertTrue(os.path.islink(image_link))
    self.assertEqual(os.readlink(image_link), image)

    image2 = os.path.join(image_repository, self.fake_image2_md5sum)
    image2_link = os.path.join(image_repository, 'image_002')
    self.assertTrue(os.path.exists(image2))
    with open(image2, 'rb') as fh:
      image2_md5sum = hashlib.md5(fh.read()).hexdigest()
    self.assertEqual(image2_md5sum, self.fake_image2_md5sum)
    self.assertTrue(os.path.islink(image2_link))
    self.assertEqual(os.readlink(image2_link), image2)

    def getRunningImageList():
      running_image_list = []
      with self.slap.instance_supervisor_rpc as instance_supervisor:
        kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
                   if 'kvm-' in q['name']][0]['pid']
        kvm_process = psutil.Process(kvm_pid)
        software_root = '/'.join([
          self.slap.software_directory,
          hashlib.md5(self.getSoftwareURL().encode('utf-8')).hexdigest()])
        for entry in kvm_process.cmdline():
          if entry.startswith('file') and 'media=cdrom' in entry:
            # do cleanups
            entry = entry.replace(software_root, '')
            entry = entry.replace(kvm_instance_partition, '')
            running_image_list.append(entry)
      return running_image_list

    # mimic the requirement: restart the instance by requesting it stopped and
    # then started started, like user have to do it
    self.rerequestInstance(partition_parameter_kw, state='stopped')
    self.slap.waitForInstance(max_retry=1)
    self.rerequestInstance(partition_parameter_kw, state='started')
    self.slap.waitForInstance(max_retry=3)

    self.assertEqual(
      [
        'file=/srv/%s/image_001,media=cdrom' % (self.image_directory,),
        'file=/srv/%s/image_002,media=cdrom' % (self.image_directory,),
        'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
        'media=cdrom'
      ],
      getRunningImageList()
    )

    # cleanup of images works, also asserts that configuration changes are
    # reflected
    partition_parameter_kw[self.key] = ''
    self.rerequestInstance(partition_parameter_kw)
    self.slap.waitForInstance(max_retry=2)
    self.assertEqual(
      os.listdir(image_repository),
      []
    )

    # mimic the requirement: restart the instance by requesting it stopped and
    # then started started, like user have to do it
    self.rerequestInstance(partition_parameter_kw, state='stopped')
    self.slap.waitForInstance(max_retry=1)
    self.rerequestInstance(partition_parameter_kw, state='started')
    self.slap.waitForInstance(max_retry=3)

    # again only default image is available in the running process
    self.assertEqual(
      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
       'media=cdrom'],
      getRunningImageList()
    )

  def assertPromiseFails(self, promise):
    partition_directory = os.path.join(
      self.slap.instance_directory,
      self.kvm_instance_partition_reference)
    monitor_run_promise = os.path.join(
      partition_directory, 'software_release', 'bin',
      'monitor.runpromise'
    )
    monitor_configuration = os.path.join(
      partition_directory, 'etc', 'monitor.conf')

    self.assertNotEqual(
      0,
      subprocess.call([
        monitor_run_promise, '-c', monitor_configuration, '-a', '-f',
        '--run-only', promise])
    )

  def test_bad_parameter(self):
    self.rerequestInstance({
      self.key: self.bad_value
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)

  def test_incorrect_md5sum(self):
    self.rerequestInstance({
      self.key: self.incorrect_md5sum_value_image % (self.fake_image,)
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)
    self.rerequestInstance({
      self.key: self.incorrect_md5sum_value
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)

  def test_not_matching_md5sum(self):
    self.rerequestInstance({
      self.key: self.single_image_value % (
        self.fake_image, self.fake_image_wrong_md5sum)
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.download_md5sum_promise)
    self.assertPromiseFails(self.download_state_promise)

  def test_unreachable_host(self):
    self.rerequestInstance({
      self.key: self.unreachable_host_value % (
        self.fake_image_md5sum,)
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.download_state_promise)

  def test_too_many_images(self):
    self.rerequestInstance({
      self.key: self.too_many_image_value
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)


@skipUnlessKvm
class TestBootImageUrlListResilient(TestBootImageUrlList):
  kvm_instance_partition_reference = 'biul2'
  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'


@skipUnlessKvm
class TestBootImageUrlSelect(TestBootImageUrlList):
  __partition_reference__ = 'bius'
  kvm_instance_partition_reference = 'bius0'

  # variations
  key = 'boot-image-url-select'
  test_input = '["%s#%s", "%s#%s"]'
  image_directory = 'boot-image-url-select-repository'
  config_state_promise = 'boot-image-url-select-config-state-promise.py'
  download_md5sum_promise = 'boot-image-url-select-download-md5sum-promise.py'
  download_state_promise = 'boot-image-url-select-download-state-promise.py'

  bad_value = '["jsutbad"]'
  incorrect_md5sum_value_image = '["%s#"]'
  incorrect_md5sum_value = '["url#asdasd"]'
  single_image_value = '["%s#%s"]'
  unreachable_host_value = '["evennotahost#%s"]'
  too_many_image_value = """[
      "image1#11111111111111111111111111111111",
      "image2#22222222222222222222222222222222",
      "image3#33333333333333333333333333333333",
      "image4#44444444444444444444444444444444",
      "image5#55555555555555555555555555555555",
      "image6#66666666666666666666666666666666"
      ]"""

  def test_not_json(self):
    self.rerequestInstance({
      self.key: 'notjson#notjson'
    })
    self.raising_waitForInstance(3)
    self.assertPromiseFails(self.config_state_promise)

  def test_together(self):
    partition_parameter_kw = {
      'boot-image-url-list': "%s#%s" % (
        self.fake_image, self.fake_image_md5sum),
      'boot-image-url-select': '["%s#%s"]' % (
        self.fake_image, self.fake_image_md5sum)
    }
    self.rerequestInstance(partition_parameter_kw)
    self.slap.waitForInstance(max_retry=10)
    # check that image is correctly downloaded and linked
    for image_directory in [
      'boot-image-url-list-repository', 'boot-image-url-select-repository']:
      image_repository = os.path.join(
        self.slap.instance_directory, self.kvm_instance_partition_reference,
        'srv', image_directory)
      image = os.path.join(image_repository, self.fake_image_md5sum)
      image_link = os.path.join(image_repository, 'image_001')
      self.assertTrue(os.path.exists(image))
      with open(image, 'rb') as fh:
        image_md5sum = hashlib.md5(fh.read()).hexdigest()
      self.assertEqual(image_md5sum, self.fake_image_md5sum)
      self.assertTrue(os.path.islink(image_link))
      self.assertEqual(os.readlink(image_link), image)

    kvm_instance_partition = os.path.join(
      self.slap.instance_directory, self.kvm_instance_partition_reference)

    def getRunningImageList():
      running_image_list = []
      with self.slap.instance_supervisor_rpc as instance_supervisor:
        kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
                   if 'kvm-' in q['name']][0]['pid']
        kvm_process = psutil.Process(kvm_pid)
        software_root = '/'.join([
          self.slap.software_directory,
          hashlib.md5(self.getSoftwareURL().encode('utf-8')).hexdigest()])
        for entry in kvm_process.cmdline():
          if entry.startswith('file') and 'media=cdrom' in entry:
            # do cleanups
            entry = entry.replace(software_root, '')
            entry = entry.replace(kvm_instance_partition, '')
            running_image_list.append(entry)
      return running_image_list

    # mimic the requirement: restart the instance by requesting it stopped and
    # then started started, like user have to do it
    self.rerequestInstance(partition_parameter_kw, state='stopped')
    self.slap.waitForInstance(max_retry=1)
    self.rerequestInstance(partition_parameter_kw, state='started')
    self.slap.waitForInstance(max_retry=3)

    self.assertEqual(
      [
        'file=/srv/boot-image-url-select-repository/image_001,media=cdrom',
        'file=/srv/boot-image-url-list-repository/image_001,media=cdrom',
        'file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
        'media=cdrom'
      ],
      getRunningImageList()
    )

    # cleanup of images works, also asserts that configuration changes are
    # reflected
    self.rerequestInstance(
      {'boot-image-url-list': '', 'boot-image-url-select': ''})
    self.slap.waitForInstance(max_retry=2)
    for image_directory in [
      'boot-image-url-list-repository', 'boot-image-url-select-repository']:
      image_repository = os.path.join(
        kvm_instance_partition, 'srv', image_directory)
      self.assertEqual(
        os.listdir(image_repository),
        []
      )

    # cleanup of images works, also asserts that configuration changes are
    # reflected
    partition_parameter_kw[self.key] = ''
    partition_parameter_kw['boot-image-url-list'] = ''
    self.rerequestInstance(partition_parameter_kw)
    self.slap.waitForInstance(max_retry=2)
    self.assertEqual(
      os.listdir(image_repository),
      []
    )

    # mimic the requirement: restart the instance by requesting it stopped and
    # then started started, like user have to do it
    self.rerequestInstance(partition_parameter_kw, state='stopped')
    self.slap.waitForInstance(max_retry=1)
    self.rerequestInstance(partition_parameter_kw, state='started')
    self.slap.waitForInstance(max_retry=3)

    # again only default image is available in the running process
    self.assertEqual(
      ['file=/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso,'
       'media=cdrom'],
      getRunningImageList()
    )


@skipUnlessKvm
class TestBootImageUrlSelectResilient(TestBootImageUrlSelect):
  kvm_instance_partition_reference = 'bius2'
  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'


@skipUnlessKvm
class TestBootImageUrlListKvmCluster(InstanceTestCase, FakeImageServerMixin):
  __partition_reference__ = 'biulkc'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  input_value = "%s#%s"
  key = 'boot-image-url-list'
  config_file_name = 'boot-image-url-list.conf'

  def setUp(self):
    super(InstanceTestCase, self).setUp()
    self.startImageHttpServer()

  def tearDown(self):
    self.stopImageHttpServer()
    super(InstanceTestCase, self).tearDown()

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True,
        },
        "KVM1": {
            "disable-ansible-promise": True,
        }
      }
    })}

  def test(self):
    # Note: As there is no way to introspect nicely where partition landed
    #       we assume ordering of the cluster requests
    self.rerequestInstance({'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True,
            self.key: self.input_value % (
              self.fake_image, self.fake_image_md5sum)
        },
        "KVM1": {
            "disable-ansible-promise": True,
            self.key: self.input_value % (
              self.fake_image2, self.fake_image2_md5sum)
        }
      }
    })})
    self.slap.waitForInstance(max_retry=10)
    KVM0_config = os.path.join(
      self.slap.instance_directory, self.__partition_reference__ + '1', 'etc',
      self.config_file_name)
    KVM1_config = os.path.join(
      self.slap.instance_directory, self.__partition_reference__ + '2', 'etc',
      self.config_file_name)
    with open(KVM0_config, 'r') as fh:
      self.assertEqual(
        self.input_value % (self.fake_image, self.fake_image_md5sum),
        fh.read().strip()
      )
    with open(KVM1_config, 'r') as fh:
      self.assertEqual(
        self.input_value % (self.fake_image2, self.fake_image2_md5sum),
        fh.read().strip()
      )


@skipUnlessKvm
class TestBootImageUrlSelectKvmCluster(TestBootImageUrlListKvmCluster):
  __partition_reference__ = 'biuskc'

  input_value = "[\"%s#%s\"]"
  key = 'boot-image-url-select'
  config_file_name = 'boot-image-url-select.json'


@skipUnlessKvm
class TestCpuMemMaxDynamic(InstanceTestCase):
  __partition_reference__ = 'cmm'

  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'cpu-count': 2,
      'ram-size': 2048
    }

  def test(self):
    with open(os.path.join(
     self.computer_partition_root_path, 'bin', 'kvm_raw'), 'r') as fh:
      kvm_raw = fh.read()
    self.assertIn('smp_count = 2', kvm_raw)
    self.assertIn('smp_max_count = 3', kvm_raw)
    self.assertIn('ram_size = 2048', kvm_raw)
    self.assertIn("ram_max_size = '2560'", kvm_raw)


@skipUnlessKvm
class TestNatRules(InstanceTestCase):
  __partition_reference__ = 'nr'

  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'nat-rules': '100 200',
    }

  def test(self):
    connection_parameter_dict = self.computer_partition\
      .getConnectionParameterDict()

    self.assertIn('nat-rule-port-tcp-100', connection_parameter_dict)
    self.assertIn('nat-rule-port-tcp-200', connection_parameter_dict)

    self.assertEqual(
      '%s : 10100' % (self._ipv6_address,),
      connection_parameter_dict['nat-rule-port-tcp-100']
    )
    self.assertEqual(
      '%s : 10200' % (self._ipv6_address,),
      connection_parameter_dict['nat-rule-port-tcp-200']
    )


@skipUnlessKvm
class TestNatRulesKvmCluster(InstanceTestCase):
  __partition_reference__ = 'nrkc'

  nat_rules = ["100", "200", "300"]
  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "nat-rules": cls.nat_rules,
            "disable-ansible-promise": True,
        }
      }
    })}

  def getRunningHostFwd(self):
    with self.slap.instance_supervisor_rpc as instance_supervisor:
      kvm_pid = [q for q in instance_supervisor.getAllProcessInfo()
                 if 'kvm-' in q['name']][0]['pid']
      kvm_process = psutil.Process(kvm_pid)
      for entry in kvm_process.cmdline():
        if 'hostfwd' in entry:
          return entry

  def test(self):
    host_fwd_entry = self.getRunningHostFwd()
    self.assertIn(
      'hostfwd=tcp:%s:10100-:100' % (self._ipv4_address,),
      host_fwd_entry)
    self.assertIn(
      'hostfwd=tcp:%s:10200-:200' % (self._ipv4_address,),
      host_fwd_entry)
    self.assertIn(
      'hostfwd=tcp:%s:10300-:300' % (self._ipv4_address,),
      host_fwd_entry)


@skipUnlessKvm
class TestNatRulesKvmClusterComplex(TestNatRulesKvmCluster):
  __partition_reference__ = 'nrkcc'
  nat_rules = ["100", "200 300"]


@skipUnlessKvm
class TestWhitelistFirewall(InstanceTestCase):
  __partition_reference__ = 'wf'
  kvm_instance_partition_reference = 'wf0'

  def test(self):
    slapos_whitelist_firewall = os.path.join(
      self.slap.instance_directory, self.kvm_instance_partition_reference,
      '.slapos-whitelist-firewall')
    self.assertTrue(os.path.exists(slapos_whitelist_firewall))
    with open(slapos_whitelist_firewall, 'rb') as fh:
      content = fh.read()
    try:
      self.content_json = json.loads(content)
    except ValueError:
      self.fail('Failed to parse json of %r' % (content,))
    self.assertTrue(isinstance(self.content_json, list))
    # check /etc/resolv.conf
    with open('/etc/resolv.conf', 'r') as f:
      resolv_conf_ip_list = []
      for line in f.readlines():
        if line.startswith('nameserver'):
          resolv_conf_ip_list.append(line.split()[1])
    resolv_conf_ip_list = list(set(resolv_conf_ip_list))
    self.assertFalse(len(resolv_conf_ip_list) == 0)
    self.assertTrue(all([q in self.content_json for q in resolv_conf_ip_list]))
    # there is something more
    self.assertGreater(len(self.content_json), len(resolv_conf_ip_list))


@skipUnlessKvm
class TestWhitelistFirewallRequest(TestWhitelistFirewall):
  whitelist_domains = '2.2.2.2 3.3.3.3\n4.4.4.4'
  @classmethod
  def getInstanceParameterDict(cls):
    return {
      'whitelist-domains': cls.whitelist_domains,
    }

  def test(self):
    super(TestWhitelistFirewallRequest, self).test()
    self.assertIn('2.2.2.2', self.content_json)
    self.assertIn('3.3.3.3', self.content_json)
    self.assertIn('4.4.4.4', self.content_json)


@skipUnlessKvm
class TestWhitelistFirewallResilient(TestWhitelistFirewall):
  kvm_instance_partition_reference = 'wf2'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'


@skipUnlessKvm
class TestWhitelistFirewallRequestResilient(TestWhitelistFirewallRequest):
  kvm_instance_partition_reference = 'wf2'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-resilient'


@skipUnlessKvm
class TestWhitelistFirewallCluster(TestWhitelistFirewall):
  kvm_instance_partition_reference = 'wf1'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "disable-ansible-promise": True
        }
      }
    })}


@skipUnlessKvm
class TestWhitelistFirewallRequestCluster(TestWhitelistFirewallRequest):
  kvm_instance_partition_reference = 'wf1'

  @classmethod
  def getInstanceSoftwareType(cls):
    return 'kvm-cluster'

  @classmethod
  def getInstanceParameterDict(cls):
    return {'_': json.dumps({
      "kvm-partition-dict": {
        "KVM0": {
            "whitelist-domains": cls.whitelist_domains,
            "disable-ansible-promise": True
        }
      }
    })}


@skipUnlessKvm
class TestDiskDevicePathWipeDiskOndestroy(InstanceTestCase, KvmMixin):
  __partition_reference__ = 'ddpwdo'
  kvm_instance_partition_reference = 'ddpwdo0'

  def test(self):
    self.rerequestInstance({
      'disk-device-path': '/dev/virt0 /dev/virt1',
      'wipe-disk-ondestroy': True
    })
    self.raising_waitForInstance(3)
    instance_path = os.path.join(
      self.slap.instance_directory, self.kvm_instance_partition_reference)

    slapos_wipe_device_disk = os.path.join(
      instance_path, 'etc', 'prerm', 'slapos_wipe_device_disk')

    # check prerm script, it's trusted that prerm manager really works
    self.assertTrue(os.path.exists(slapos_wipe_device_disk))
    with open(slapos_wipe_device_disk) as fh:
      self.assertEqual(
        fh.read().strip(),
        r"""dd if=/dev/zero of=/dev/virt0 bs=4096 count=500k
dd if=/dev/zero of=/dev/virt1 bs=4096 count=500k"""
      )
    self.assertTrue(os.access(slapos_wipe_device_disk, os.X_OK))


@skipUnlessKvm
class TestImageDownloadController(InstanceTestCase, FakeImageServerMixin):
  __partition_reference__ = 'idc'
  maxDiff = None

  def setUp(self):
    super(TestImageDownloadController, self).setUp()
    self.working_directory = tempfile.mkdtemp()
    self.destination_directory = os.path.join(
      self.working_directory, 'destination')
    os.mkdir(self.destination_directory)
    self.config_json = os.path.join(
      self.working_directory, 'config.json')
    self.md5sum_fail_file = os.path.join(
      self.working_directory, 'md5sum_fail_file')
    self.error_state_file = os.path.join(
      self.working_directory, 'error_state_file')
    self.processed_md5sum = os.path.join(
      self.working_directory, 'processed_md5sum')
    self.startImageHttpServer()
    self.image_download_controller = os.path.join(
      self.slap.instance_directory, self.__partition_reference__ + '0',
      'software_release', 'parts', 'image-download-controller',
      'image-download-controller')

  def tearDown(self):
    self.stopImageHttpServer()
    shutil.rmtree(self.working_directory)
    super(InstanceTestCase, self).tearDown()

  def callImageDownloadController(self, *args):
    call_list = [sys.executable, self.image_download_controller] + list(args)
    try:
      return (0, subprocess.check_output(
        call_list, stderr=subprocess.STDOUT).decode('utf-8'))
    except subprocess.CalledProcessError as e:
      return (e.returncode, e.output.decode('utf-8'))

  def runImageDownloadControlerWithDict(self, json_dict):
    with open(self.config_json, 'w') as fh:
      json.dump(json_dict, fh, indent=2)
    return self.callImageDownloadController(
      self.config_json,
      'curl',  # comes from test environemnt, considered to be recent enough
      self.md5sum_fail_file,
      self.error_state_file,
      self.processed_md5sum
    )

  def assertFileContent(self, path, content):
    self.assertTrue(os.path.exists, path)
    with open(path, 'r') as fh:
      self.assertEqual(
        fh.read(),
        content)

  def test(self):
    json_dict = {
      'error-amount': 0,
      'config-md5sum': 'config-md5sum',
      'destination-directory': self.destination_directory,
      'image-list': [
        {
          'destination-tmp': 'tmp',
          'url': self.fake_image,
          'destination': 'destination',
          'link': 'image_001',
          'gzipped': False,
          'md5sum': self.fake_image_md5sum,
        }
      ]
    }
    code, result = self.runImageDownloadControlerWithDict(
      json_dict
    )
    self.assertEqual(
      (code, result.strip()),
      (0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
INF: %(fake_image)s : Stored with checksum %(checksum)s
INF: %(fake_image)s : Symlinking %(symlink)s -> %(destination)s
""".strip() % {
        'fake_image': self.fake_image,
        'checksum': self.fake_image_md5sum,
        'error_state_file': self.error_state_file,
        'symlink': os.path.join(self.destination_directory, 'image_001'),
        'destination': os.path.join(self.destination_directory, 'destination'),
      })
    )
    self.assertFileContent(self.md5sum_fail_file, '')
    self.assertFileContent(self.error_state_file, '')
    self.assertFileContent(self.processed_md5sum, 'config-md5sum')
    self.assertFalse(
      os.path.exists(os.path.join(self.destination_directory, 'tmp')))
    self.assertFileContent(
      os.path.join(self.destination_directory, 'destination'),
      'fake_image_content'
    )

    # Nothing happens if all is downloaded
    code, result = self.runImageDownloadControlerWithDict(
      json_dict
    )
    self.assertEqual(
      (code, result.strip()),
      (0, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : already downloaded
""".strip() % {
        'fake_image': self.fake_image,
        'checksum': self.fake_image_md5sum,
        'error_state_file': self.error_state_file,
        'symlink': os.path.join(self.destination_directory, 'image_001'),
        'destination': os.path.join(self.destination_directory, 'destination'),
      })
    )

  def test_fail(self):
    json_dict = {
      'error-amount': 0,
      'config-md5sum': 'config-md5sum',
      'destination-directory': self.destination_directory,
      'image-list': [
        {
          'destination-tmp': 'tmp',
          'url': self.fake_image,
          'destination': 'destination',
          'link': 'image_001',
          'gzipped': False,
          'md5sum': self.fake_image_wrong_md5sum,
        }
      ]
    }
    for try_num in range(1, 5):
      code, result = self.runImageDownloadControlerWithDict(
        json_dict
      )
      self.assertEqual(
        (code, result.strip()),
        (1, """
INF: Storing errors in %(error_state_file)s
INF: %(fake_image)s : Downloading
""".  strip() % {
          'fake_image': self.fake_image,
          'error_state_file': self.error_state_file,
          'symlink': os.path.join(self.destination_directory, 'image_001'),
          'destination': os.path.join(
            self.destination_directory, 'destination'),
        })
      )
      fake_image_url = '#'.join([
        self.fake_image, self.fake_image_wrong_md5sum])
      self.assertFileContent(
        self.md5sum_fail_file, """{
  "%s": %s
}""" % (fake_image_url, try_num))
      self.assertFileContent(
        self.error_state_file, """
        ERR: %(fake_image)s : MD5 mismatch expected is %(wrong_checksum)s """
        """but got instead %(real_checksum)s""".strip() % {
          'fake_image': self.fake_image,
          'wrong_checksum': self.fake_image_wrong_md5sum,
          'real_checksum': self.fake_image_md5sum,
        })
      self.assertFileContent(self.processed_md5sum, 'config-md5sum')
      self.assertFalse(
        os.path.exists(os.path.join(self.destination_directory, 'tmp')))
      self.assertFalse(
        os.path.exists(
          os.path.join(self.destination_directory, 'destination')))

    code, result = self.runImageDownloadControlerWithDict(
      json_dict
    )
    self.assertEqual(
      (code, result.strip()),
      (1, """
INF: Storing errors in %(error_state_file)s
""".  strip() % {
        'fake_image': self.fake_image,
        'error_state_file': self.error_state_file,
        'symlink': os.path.join(self.destination_directory, 'image_001'),
        'destination': os.path.join(
          self.destination_directory, 'destination'),
      })
    )
    fake_image_url = '#'.join([
      self.fake_image, self.fake_image_wrong_md5sum])
    self.assertFileContent(
      self.md5sum_fail_file, """{
  "%s": %s
}""" % (fake_image_url, 4))
    self.assertFileContent(
      self.error_state_file, """
      ERR: %(fake_image)s : Checksum is incorrect after 4 tries, will not """
      """retry""".strip() % {
        'fake_image': self.fake_image,
      })
    self.assertFileContent(self.processed_md5sum, 'config-md5sum')
    self.assertFalse(
      os.path.exists(os.path.join(self.destination_directory, 'tmp')))
    self.assertFalse(
      os.path.exists(
        os.path.join(self.destination_directory, 'destination')))