Adapt build with buildout3

obs/slapos/prepare_download_cache.sh

 #!/bin/bash
 
 set -e
+set -o pipefail
 
 source release_configuration.sh
 
@@ -23,27 +24,29 @@ echo " Buildroot Directory: $BUILD_ROOT_DIRECTORY "
 # build the package once keeping every source in cache #
 ########################################################
 
-mkdir -p $BUILD_DIRECTORY/{eggs,extends-cache,download-cache/dist}
-cd $BUILD_DIRECTORY
-
-# 1) boostrap with isolation
-echo "bootsrapping buildout"
-sed  "s/\%RECIPE_VERSION\%/$RECIPE_VERSION/g;s|\%PATCHES_DIRECTORY\%|$PATCHES_DIRECTORY|g;s|\%TARGET_DIRECTORY\%|$TARGET_DIRECTORY|g;s|\%BUILD_ROOT_DIRECTORY\%|$BUILD_ROOT_DIRECTORY|g;s|\%BUILD_DIRECTORY\%|$BUILD_DIRECTORY|g" $BUILD_ROOT_DIRECTORY/../slapos.buildout.cfg.in > buildout.cfg
+mkdir -p $BUILD_DIRECTORY/{eggs,extends-cache,download-cache/dist,bootstrap-dir}
+cd $BUILD_DIRECTORY/bootstrap-dir
 
-buildout bootstrap buildout:isolate-from-buildout-and-setuptools-path=true
-ls download-cache/dist/*.whl && { echo "There shouldn't be any wheel in download-cache" ; exit 1 ; }
+# 1) you need a recent enough version of Nexedi buildout, you can do e.g.:
+# pip install http://www.nexedi.org/static/packages/source/slapos.buildout/zc.buildout-3.0.1%2Bslapos002.tar.gz
 
 # 2) compile very simple buildout with networkcache
 echo "Preparing networkcached zc.buildout"
 sed  "s/\%RECIPE_VERSION\%/$RECIPE_VERSION/g;s|\%PATCHES_DIRECTORY\%|$PATCHES_DIRECTORY|g;s|\%TARGET_DIRECTORY\%|$TARGET_DIRECTORY|g;s|\%BUILD_ROOT_DIRECTORY\%|$BUILD_ROOT_DIRECTORY|g;s|\%BUILD_DIRECTORY\%|$BUILD_DIRECTORY|g" $BUILD_ROOT_DIRECTORY/../networkcached.cfg.in > buildout.cfg
-sed -i '1s/$/ -S/' bin/buildout
-bin/buildout buildout:newest=true -v
+# be careful to let 'extra-paths=' in buildout section for isolation
+buildout buildout:download-cache=../download-cache bootstrap
 
-# 3) build locally everything with gcc to get download-cache and extends-cache ready
-echo "Launch the big buildout to compile everything"
+# 3) bootstrap the buildout used for full compilation
+echo "Launch the bootstrap of final buildout"
+cd $BUILD_DIRECTORY
 sed  "s/\%RECIPE_VERSION\%/$RECIPE_VERSION/g;s|\%PATCHES_DIRECTORY\%|$PATCHES_DIRECTORY|g;s|\%TARGET_DIRECTORY\%|$TARGET_DIRECTORY|g;s|\%BUILD_ROOT_DIRECTORY\%|$BUILD_ROOT_DIRECTORY|g;s|\%BUILD_DIRECTORY\%|$BUILD_DIRECTORY|g" $BUILD_ROOT_DIRECTORY/../buildout_with_gcc.cfg.in > buildout.cfg
+bootstrap-dir/bin/buildout bootstrap
+
+# 4) build locally everything with gcc to get download-cache and extends-cache ready
+echo "Launch the big buildout to compile everything"
+bin/buildout buildout:newest=true -v | tee buildout-full.log
+
 
-bin/buildout buildout:newest=true -v
 
 ###################################################
 # remove all files from build keeping only caches #
@@ -68,11 +71,10 @@ find . -regextype posix-extended -type f \
 
 #TODO remove git files
 
+
 ##################################
 # prepare compilation inside OBS #
 ##################################
-# we need the very first bootstrap script
-cp $CURRENT_DIRECTORY/../_generic/compilation/makefile-scripts/bootstrap $BUILD_DIRECTORY
 
 # we need the original directory to do a sed inside OBS
 # TODO remove this and properly use extends-cache instead
@@ -81,3 +83,4 @@ echo "$BUILD_ROOT_DIRECTORY" > $CURRENT_DIRECTORY/$SLAPOS_DIRECTORY/slapos/origi
 # in OBS build, don't force gcc build
 sed  "s/\%RECIPE_VERSION\%/$RECIPE_VERSION/g;s|\%PATCHES_DIRECTORY\%|$PATCHES_DIRECTORY|g;s|\%TARGET_DIRECTORY\%|$TARGET_DIRECTORY|g;s|\%BUILD_ROOT_DIRECTORY\%|$BUILD_ROOT_DIRECTORY|g;s|\%BUILD_DIRECTORY\%|$BUILD_DIRECTORY|g" $BUILD_ROOT_DIRECTORY/../buildout_without_gcc.cfg.in > $BUILD_DIRECTORY/buildout.cfg
 
+

obs/slapos/slapos-node/Makefile

@@ -2,7 +2,7 @@
 # -*- makefile -*-
 
 build: 
-	make -C slapos build SLAPOS_BOOTSTRAP_SYSTEM_PYTHON=$(shell which python2.7 2> /dev/null || echo python3)
+	make -C slapos build SLAPOS_BOOTSTRAP_SYSTEM_PYTHON=$(shell which python3 2> /dev/null || echo python2.7)
 
 all: build
 

obs/slapos/slapos-node/slapos/buildout_without_gcc.cfg.in

@@ -2,6 +2,7 @@
 rootdir = %TARGET_DIRECTORY%
 destdir = %BUILD_ROOT_DIRECTORY% 
 builddir = %BUILD_DIRECTORY%
+extra-paths =
 extends =
   %BUILD_ROOT_DIRECTORY%/../slapos_repository/component/slapos/obs.cfg
 download-cache = download-cache

obs/slapos/slapos-node/slapos/networkcached.cfg.in

@@ -2,6 +2,7 @@
 rootdir = %TARGET_DIRECTORY%
 destdir = %BUILD_ROOT_DIRECTORY%
 builddir = %BUILD_DIRECTORY%
+extra-paths =
 extensions =
 extends-cache = extends-cache
 download-cache = download-cache
@@ -13,6 +14,8 @@ find-links +=
   http://www.nexedi.org/static/packages/source/
   http://www.nexedi.org/static/packages/source/slapos.buildout/
 
+allow-picked-versions = false
+
 [networkcached]
 recipe = zc.recipe.egg
 eggs =
@@ -20,7 +23,13 @@ eggs =
   zc.buildout
 
 [versions]
-setuptools = 44.1.1
-zc.buildout = 2.7.1+slapos020
+# The last version of setuptools compatible with Python 3.7
+setuptools = 67.8.0
+# Use SlapOS patched zc.buildout
+zc.buildout = 3.0.1+slapos002
+pip = 23.2.1
+# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
+zc.recipe.egg = 2.0.8.dev0+slapos002
+wheel = 0.41.2:whl
+
 slapos.libnetworkcache = 0.25
-zc.recipe.egg = 2.0.3+slapos003

obs/slapos/slapos-node/slapos/slapos.buildout.cfg.in

@@ -19,6 +19,11 @@ eggs =
   zc.buildout
 
 [versions]
-setuptools = 44.1.1
-zc.buildout = 2.7.1+slapos020
-zc.recipe.egg = 2.0.3+slapos003
+# The last version of setuptools compatible with Python 3.7
+setuptools = 67.8.0
+# Use SlapOS patched zc.buildout
+zc.buildout = 3.0.1+slapos002
+pip = 23.2.1
+# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
+zc.recipe.egg = 2.0.8.dev0+slapos002
+wheel = 0.41.2

obs/slapos/templates/Makefile.in

@@ -23,7 +23,7 @@ build-stamp:
 
 	@echo "Bootstrapping buildout"
 	cd $(BUILD_DIRECTORY) && \
-		$(SLAPOS_BOOTSTRAP_SYSTEM_PYTHON) ./bootstrap
+		$(SLAPOS_BOOTSTRAP_SYSTEM_PYTHON) ./bootstrap-dir/bin/buildout buildout:install-from-cache=true bootstrap
 
 	@echo 'Preparing source tarball (recipe version: $(RECIPE_VERSION))'
 	cd $(BUILD_DIRECTORY) && \