diff --git a/software/slapos-testing/buildout.hash.cfg b/software/slapos-testing/buildout.hash.cfg
index c13ced29aab5b01a0669024480bf378170b0a0bb..27769171716fe17a013584fde4d5a5fc94968932 100644
--- a/software/slapos-testing/buildout.hash.cfg
+++ b/software/slapos-testing/buildout.hash.cfg
@@ -15,4 +15,4 @@
 
 [template]
 filename = instance.cfg
-md5sum = 2a0edca6e0aa2c1277747c980bc0f256
+md5sum = 84bd6729e9b299c457cea2d1be6d05a4
diff --git a/software/slapos-testing/instance.cfg b/software/slapos-testing/instance.cfg
index 3505139c5241ad042f10e656c72284fee52569ae..759938f0af11e112a188e4f54147a077d15c151d 100644
--- a/software/slapos-testing/instance.cfg
+++ b/software/slapos-testing/instance.cfg
@@ -32,6 +32,16 @@ repository = ${kedifa-repository:location}
 <= download-source
 repository = ${caucase-repository:location}
 
+[caucase-test-runner]
+recipe = slapos.recipe.template:jinja2
+template = inline:#!/bin/sh
+  export HOSTS="$(mktemp)"
+  trap 'rm "$HOSTS"' EXIT
+  printf '%s testhost\n%s testhost\n' "$SLAPOS_TEST_IPV4" "$SLAPOS_TEST_IPV6" > "$HOSTS"
+  export CAUCASE_NETLOC=testhost:8000 LD_PRELOAD=${userhosts:location}/lib/userhosts.so:$LD_PRELOAD
+  exec python -m unittest discover -v
+rendered = $${caucase:location}/host_setting.sh
+
 [slapos.libnetworkcache]
 <= download-source
 repository = ${slapos.libnetworkcache-repository:location}
@@ -98,9 +108,7 @@ template = inline:
   )
   TestCase(
       "caucase",
-      # XXX caucase uses 2to3 dynamically in setup.py, so it only supports
-      # runnning tests with python setup.py test
-      ['python', 'setup.py', 'test'],
+      ['$${caucase-test-runner:rendered}'],
       cwd="""$${caucase:location}""",
       summaryf=UnitTest.summary,
   )
diff --git a/software/slapos-testing/software.cfg b/software/slapos-testing/software.cfg
index 9a4e9a33ee96d354b41e855c27ec125993136197..ebdc0ee42216f92a8ea8a8f9abc6a682482c32e3 100644
--- a/software/slapos-testing/software.cfg
+++ b/software/slapos-testing/software.cfg
@@ -16,6 +16,7 @@ extends =
   ../../component/jq/buildout.cfg
   ../../component/sed/buildout.cfg
   ../../component/grep/buildout.cfg
+  ../../component/userhosts/buildout.cfg
   ../../stack/slapos.cfg
   ../../stack/caucase/buildout.cfg
   ../../stack/nxdtest.cfg