version up: zc.buildout, slapos.recipe.build, Debian 10/11 netinst

Thanks to the new buildout/slapos.recipe.build feature,
we can download the last released version of Debian Stable.

NEO stress testing is changed to run on both Debian 10 & 11.

Keep proposing Debian 10 netinst in kvm SR,
waiting that Debian 11 is more tested.

component/qemu-kvm/buildout.cfg

@@ -63,9 +63,10 @@ environment +=
 
 [debian-netinst-base]
 recipe = slapos.recipe.build:download
-filename = ${:_buildout_section_name_}
-url = https://cdimage.debian.org/cdimage/${:release}/${:arch}/iso-cd/debian-${:version}-${:arch}-netinst.iso
-release = archive/${:version}
+shared = true
+filename = debian-${:version}-${:arch}-netinst.iso
+url = https://cdimage.debian.org/cdimage/archive/${:archive}/${:arch}/iso-cd/${:filename}
+archive = ${:version}
 
 [debian-amd64-netinst-base]
 <= debian-netinst-base
@@ -86,16 +87,23 @@ md5sum = df0ce86d0b1d81e232ad08eef58754ed
 version = 9.13.0
 md5sum = 6097fdb9cbab47c96471274b9044e983
 
-[debian-amd64-netinst.iso]
-# Download the installer of Debian 10 (Buster)
-# XXX: This is not the latest version because
-#      Debian does not provide a stable URL for it.
+[debian-amd64-buster-netinst.iso]
 <= debian-amd64-netinst-base
 version = 10.10.0
 md5sum = c7d0e562e589e853b5d00563b4311720
 
+[debian-amd64-bullseye-netinst.iso]
+<= debian-amd64-netinst-base
+version = 11.0.0
+md5sum = 499953266841cae41612310e65659456
+alternate-url = https://cdimage.debian.org/cdimage/release/current/${:arch}/iso-cd/${:filename}
+
+[debian-amd64-netinst.iso]
+<= debian-amd64-bullseye-netinst.iso
+
 [debian-amd64-testing-netinst.iso]
 <= debian-amd64-netinst-base
-release = bullseye_di_rc3
+alternate-url = https://cdimage.debian.org/cdimage/${archive}/${:arch}/iso-cd/${:filename}
+archive = bullseye_di_rc3
 version = bullseye-DI-rc3
 md5sum = 405917de7062c58357a3673c9901f0c4

component/vm-img/debian.cfg

@@ -23,7 +23,7 @@ wait-ssh = 100
 [vm-debian]
 recipe = slapos.recipe.build:vm.install-debian
 environment = vm-install-environment
-dists = debian-stable
+dists = debian-buster
 size = 2Gi
 preseed.partman/early_command =
   set -e /usr/lib/apt-setup/generators/99 AllowUnauthenticated; if [ "`debconf-get mirror/http/hostname`" = archive.debian.org ]; then echo "echo 'APT::Get::$2 \"true\";' >/target/etc/apt/apt.conf.d/01$2" >$1$2; chmod +x $1$2; fi
@@ -75,6 +75,8 @@ x86_64.iso = debian-amd64-stretch-netinst.iso
 
 [debian-buster]
 <= debian-stable
+x86_64.iso = debian-amd64-buster-netinst.iso
 
 [debian-bullseye]
-<= debian-testing
+<= debian-stable
+x86_64.iso = debian-amd64-bullseye-netinst.iso

software/kvm/buildout.hash.cfg

@@ -15,7 +15,7 @@
 
 [template]
 filename = instance.cfg.in
-md5sum = 0d9d24ef59248da8fa47535f54b83ba8
+md5sum = b129c9b2a614563d3f7c3f9e906d59f2
 
 [template-kvm]
 filename = instance-kvm.cfg.jinja2

software/kvm/instance.cfg.in

@@ -82,7 +82,7 @@ extra-context =
   raw dash_executable_location ${dash:location}/bin/dash
   raw dnsresolver_executable ${buildout:bin-directory}/dnsresolver
   raw dcron_executable_location ${dcron:location}/sbin/crond
-  raw debian_amd64_netinst_location ${debian-amd64-netinst.iso:target}
+  raw debian_amd64_netinst_location ${debian-amd64-buster-netinst.iso:target}
   raw whitelist_domains_default ${whitelist-domains-default:location}/${whitelist-domains-default:filename}
   raw whitelist_firewall_download_controller ${whitelist-firewall-download-controller:target}
   raw image_download_controller ${image-download-controller:target}

software/kvm/test/test.py

@@ -816,13 +816,13 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
 
   def getRunningImageList(self, kvm_instance_partition,
       _match_cdrom=re.compile('file=(.+),media=cdrom$').match,
+      _sub_iso=re.compile(r'(/debian)(-[^-/]+)(-[^/]+-netinst\.iso)$').sub,
       ):
     with self.slap.instance_supervisor_rpc as instance_supervisor:
       kvm_pid = next(q for q in instance_supervisor.getAllProcessInfo()
                        if 'kvm-' in q['name'])['pid']
-    software_root = os.path.join(
-      self.slap.software_directory,
-      hashlib.md5(self.getSoftwareURL().encode()).hexdigest())
+    sub_shared = re.compile(r'^%s/[^/]+/[0-9a-f]{32}/'
+                            % re.escape(self.slap.shared_directory)).sub
     image_list = []
     for entry in psutil.Process(kvm_pid).cmdline():
       m = _match_cdrom(entry)
@@ -831,9 +831,10 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
         st = os.stat(path)
         if stat.S_ISREG(st.st_mode) and st.st_size:
           image_list.append(
+            _sub_iso(r'\1-${ver}\3',
+            sub_shared(r'${shared}/',
             path.replace(kvm_instance_partition, '${inst}')
-                .replace(software_root, '${soft}')
-          )
+          )))
     return image_list
 
   def test(self):
@@ -878,7 +879,7 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
       [
         '${inst}/srv/%s/image_001' % self.image_directory,
         '${inst}/srv/%s/image_002' % self.image_directory,
-        '${soft}/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso',
+        '${shared}/debian-${ver}-amd64-netinst.iso',
       ],
       self.getRunningImageList(kvm_instance_partition)
     )
@@ -902,7 +903,7 @@ class TestBootImageUrlList(InstanceTestCase, FakeImageServerMixin):
 
     # again only default image is available in the running process
     self.assertEqual(
-      ['${soft}/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso'],
+      ['${shared}/debian-${ver}-amd64-netinst.iso'],
       self.getRunningImageList(kvm_instance_partition)
     )
 
@@ -1048,7 +1049,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
       [
         '${inst}/srv/boot-image-url-select-repository/image_001',
         '${inst}/srv/boot-image-url-list-repository/image_001',
-        '${soft}/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso',
+        '${shared}/debian-${ver}-amd64-netinst.iso',
       ],
       self.getRunningImageList(kvm_instance_partition)
     )
@@ -1087,7 +1088,7 @@ class TestBootImageUrlSelect(TestBootImageUrlList):
 
     # again only default image is available in the running process
     self.assertEqual(
-      ['${soft}/parts/debian-amd64-netinst.iso/debian-amd64-netinst.iso'],
+      ['${shared}/debian-${ver}-amd64-netinst.iso'],
       self.getRunningImageList(kvm_instance_partition)
     )
 

software/neoppod/stress-testing/runTestSuite.in

@@ -135,8 +135,9 @@ mount.neoppod = {{parameter_dict['neoppod']}}
       status_dict['stderr'] = ''.join(stderr)
 
       if not returncode:
+        os.rename(partdir, dist)
         try:
-          with open(os.path.join(partdir, 'status')) as f:
+          with open(os.path.join(dist, 'status')) as f:
             status = int(f.read())
         except Exception:
           pass
@@ -145,7 +146,7 @@ mount.neoppod = {{parameter_dict['neoppod']}}
           try:
             curses.setupterm('screen')
             rmkx = curses.tigetstr('rmkx')
-            with open(os.path.join(partdir, 'stdout')) as f:
+            with open(os.path.join(dist, 'stdout')) as f:
               stdout = f.read()
             status_dict['stdout'] = stdout[stdout.rindex(rmkx)+len(rmkx):]
           except Exception:

software/neoppod/stress-testing/software.cfg

@@ -71,6 +71,7 @@ location = ${:repository}
 [vm-debian]
 # Mainly for logs, otherwise 2G would enough (<100M for the SR).
 size = 8Gi
+dists += debian-bullseye
 late-command +=
   echo tmpfs /tmp tmpfs mode=1777,size=90% 0 0 >>/etc/fstab
 packages +=

stack/slapos.cfg

@@ -132,7 +132,7 @@ eggs =
 [versions]
 setuptools = 44.1.1
 # Use SlapOS patched zc.buildout
-zc.buildout = 2.7.1+slapos013
+zc.buildout = 2.7.1+slapos014
 # Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
 zc.recipe.egg = 2.0.3+slapos003
 # Use own version of h.r.download to be able to open .xz and .lz archives
@@ -196,7 +196,7 @@ slapos.extension.strip = 0.4
 slapos.extension.shared = 1.0
 slapos.libnetworkcache = 0.20
 slapos.rebootstrap = 4.5
-slapos.recipe.build = 0.47
+slapos.recipe.build = 0.48
 slapos.recipe.cmmi = 0.17
 slapos.recipe.template = 4.6
 slapos.toolbox = 0.124