obs/_generic: support bootstrapping with either python2 or python3

obs/_generic/build-scripts/10tarball_directory.sh

@@ -43,8 +43,6 @@ python2.7 -S bootstrap-buildout.py \
 	--buildout-version "$ZC_BUILDOUT_VERSION" \
 	--setuptools-version "$SETUPTOOLS_VERSION" \
 	--setuptools-to-dir eggs -f http://www.nexedi.org/static/packages/source/slapos.buildout/
-# backup $RUN_BUILDOUT_DIR/bin/buildout (to be restored for OBS)
-cp bin/buildout bin/backup.buildout
 # run $RUN_BUILDOUT_DIR/bin/buildout (note that it modifies itself via rebootstrapping when compiling python)
 ./bin/buildout -v
 

obs/_generic/build-scripts/20obs.sh

@@ -27,14 +27,8 @@ mv "$TARBALL_DIR/obs_buildout.cfg" "$RUN_BUILDOUT_DIR/buildout.cfg"
 echo "$TARBALL_DIR" > "$TARBALL_DIR"/local_tarball_directory_path
 # add a stamp so that OBS does not clean the local preparation before compiling
 touch "$TARBALL_DIR/clean-stamp"
-# restore bin/buildout
-# note: when installing python, buildout "rebootstraps" itself to use the installed python:
-# it modifies its own shebang, which would fail on OBS' VM (no such file or directory)
-if [ -f "$RUN_BUILDOUT_DIR"/bin/backup.buildout ]; then
-	mv "$RUN_BUILDOUT_DIR"/bin/backup.buildout "$RUN_BUILDOUT_DIR"/bin/buildout
-fi
 # clean the compilation related files
-rm -rf "$RUN_BUILDOUT_DIR"/{.installed.cfg,parts}
+rm -rf "$RUN_BUILDOUT_DIR"/{.installed.cfg,downloads,parts,eggs,develop-eggs,bin,rebootstrap}
 
 ### Prepare the archives for OBS
 # -C option allows to give tar an absolute path without archiving the directory from / (i.e. home/user/[...])
@@ -44,7 +38,7 @@ tar czf "$OBS_DIR/debian$ARCHIVE_EXT" -C "$OBS_DIR" debian
 # OBS COMMIT
 
 cd "$OBS_DIR"
-osc add *.dsc *"$ARCHIVE_EXT"
+osc add *.spec *.dsc *"$ARCHIVE_EXT"
 if [ -n "$OBS_COMMIT_MSG" ]; then
 	osc commit -m "$OBS_COMMIT_MSG"
 else

obs/_generic/compilation/makefile-scripts/bootstrap

+#!/usr/bin/python3 -S
+import glob, os, shutil, sys, tarfile, tempfile, zipfile
+dist = "download-cache/dist"
+tmp = tempfile.mkdtemp()
+try:
+    setuptools, = glob.glob(dist + "/setuptools-*")
+    if setuptools.endswith(".zip"):
+        zipfile.ZipFile(setuptools).extractall(tmp)
+    else:
+        tarfile.TarFile.open(setuptools, "r:*").extractall(tmp)
+    x, = os.listdir(tmp)
+    sys.path.insert(0, os.path.join(tmp, x))
+    from setuptools.command.easy_install import main
+    for x in "bin", "eggs":
+        try:
+            os.mkdir(x)
+        except OSError:
+            pass
+    main(["-f", dist, "-mxd", x, setuptools, "zc.buildout"])
+finally:
+  shutil.rmtree(tmp)
+
+with os.fdopen(os.open("bin/buildout", os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0o777), 'w') as f:
+    f.write("""\
+#!%s -S
+import os, sys
+d = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
+d = os.path.dirname(d) + %r
+sys.path[:0] = (%s)
+from zc.buildout.buildout import main
+sys.exit(main())
+""" % (
+    sys.executable,
+    "/%s/" % x,
+    ", ".join(map("d + %r".__mod__, os.listdir(x))),
+    ))

obs/_generic/compilation/makefile-scripts/build.sh

@@ -6,7 +6,8 @@ source makefile-scripts/compilation-env.sh
 
 echo "Fixing buildout path to "$TARBALL_DIR" rather than "$OLD_TARBALL_DIR" for buildout"
 cd "$RUN_BUILDOUT_DIR"
-sed -i "s#$OLD_TARBALL_DIR#$TARBALL_DIR#g" buildout.cfg bin/*
-bin/buildout -v
+sed -i "s#$OLD_TARBALL_DIR#$TARBALL_DIR#g" buildout.cfg
+$SLAPOS_BOOTSTRAP_SYSTEM_PYTHON $TARBALL_DIR/makefile-scripts/bootstrap
+$SLAPOS_BOOTSTRAP_SYSTEM_PYTHON bin/buildout -v
 
 touch clean-stamp

obs/_generic/compilation/makefile-scripts/compilation-env.sh.in

@@ -8,9 +8,4 @@ INSTALL_DIR="$DESTDIR""$TARGET_DIR"
 RUN_BUILDOUT_DIR="$BUILD_DIR""$TARGET_DIR"
 # get the path of the BUILD_DIR of the first build (performed to prepare the cache for OBS)
 OLD_TARBALL_DIR="$(cat local_tarball_directory_path)"
-
-TARBALL_DIR=$(realpath -m "$TARBALL_DIR")
-BUILD_DIR=$(realpath -m "$BUILD_DIR")
-INSTALL_DIR=$(realpath -m "$INSTALL_DIR")
-RUN_BUILDOUT_DIR=$(realpath -m "$RUN_BUILDOUT_DIR")
-OLD_TARBALL_DIR=$(realpath -m "$OLD_TARBALL_DIR")
+SLAPOS_BOOTSTRAP_SYSTEM_PYTHON=$(which python2.7 2> /dev/null || echo python3)

obs/fluent-bit/env.sh

@@ -3,6 +3,6 @@
 export SOFTWARE_NAME=fluent-bit
 export MAINTAINER_NAME="Ophélie Gagnard"
 export MAINTAINER_EMAIL=ophelie.gagnard@nexedi.com
-export PACKAGE_BUILD_DEPENDENCIES="debhelper, chrpath, python3 (>=3.7) | python"
+export PACKAGE_BUILD_DEPENDENCIES="debhelper, chrpath, python3 (>=3.7) | python, python3-dev (>= 3.7) | python-dev"
 export SETUPTOOLS_VERSION=44.1.1
 export ZC_BUILDOUT_VERSION=2.7.1+slapos016

obs/re6st/bootstrap

-#!/usr/bin/python3 -S
-import glob, os, shutil, sys, tarfile, tempfile, zipfile
-dist = "download-cache/dist"
-tmp = tempfile.mkdtemp()
-try:
-    setuptools, = glob.glob(dist + "/setuptools-*")
-    if setuptools.endswith(".zip"):
-        zipfile.ZipFile(setuptools).extractall(tmp)
-    else:
-        tarfile.TarFile.open(setuptools, "r:*").extractall(tmp)
-    x, = os.listdir(tmp)
-    sys.path.insert(0, os.path.join(tmp, x))
-    from setuptools.command.easy_install import main
-    for x in "bin", "eggs":
-        try:
-            os.mkdir(x)
-        except OSError:
-            pass
-    main(["-f", dist, "-mxd", x, setuptools, "zc.buildout"])
-finally:
-  shutil.rmtree(tmp)
-
-with os.fdopen(os.open("bin/buildout", os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0o777), 'w') as f:
-    f.write("""\
-#!%s -S
-import os, sys
-d = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
-d = os.path.dirname(d) + %r
-sys.path[:0] = (%s)
-from zc.buildout.buildout import main
-sys.exit(main())
-""" % (
-    sys.executable,
-    "/%s/" % x,
-    ", ".join(map("d + %r".__mod__, os.listdir(x))),
-    ))

obs/re6st/bootstrap

+../_generic/compilation/makefile-scripts/bootstrap
\ No newline at end of file

obs/slapos/prepare_download_cache.sh

@@ -84,7 +84,7 @@ find . -regextype posix-extended -type f \
 # prepare compilation inside OBS #
 ##################################
 # we need the very first bootstrap script
-cp $CURRENT_DIRECTORY/../re6st/bootstrap $BUILD_DIRECTORY
+cp $CURRENT_DIRECTORY/../_generic/compilation/makefile-scripts/bootstrap $BUILD_DIRECTORY
 
 # we need the original directory to do a sed inside OBS
 # TODO remove this and properly use extends-cache instead