Reorganize slightly and clean the build scripts.

build-scripts/:
- Do nothing after running buildout. It allows to work on other projects during the compilation time. Create a new stage.
- Rename the scripts.
- Add and rename some variables.
- Restore the buildout files retrieving using git (rather than copying a model in the current filesystem).
Makefile: Update.
- Supports both numbered and unnumbered names.
- Update for renaming and stage addition.
- Add an after_build target and fix some details.
templates/compilation-templates/: Change a maccro according to a new variable name.

obs/generic/Makefile

 
-all: build distrib_files obs
-clean: clean_build clean_distrib_files clean_obs
+all: build after_build
+clean: clean_build clean_after_build
 
-build: build_tree compilation_templates bootstrap_buildout
-clean_build: clean_build_tree clean_compilation_templates clean_bootstrap_buildout
+build: build_tree compilation_templates bootstrap_and_compile
+clean_build: clean_build_tree clean_compilation_templates clean_bootstrap_and_compile
 
-no-obs: build distrib_files
-clean_no-obs: clean_build clean_distrib_files
+after_build: backup_tarball distrib_files obs
+clean_after_build: clean_backup_tarball clean_distrib_files clean_obs
 
-build_tree:
-	build-scripts/build_tree.sh
+no-obs: build backup_tarball distrib_files
+clean_no-obs: clean_build clean_backup_tarball clean_distrib_files
+
+build_tree: 10build_tree
+10build_tree:
+	build-scripts/10build_tree.sh
 clean_build_tree:
-	build-scripts/clean_build_tree.sh
+	build-scripts/clean_10build_tree.sh
 
-compilation_templates:
-	build-scripts/compilation_templates.sh
+compilation_templates: 20compilation_templates
+20compilation_templates:
+	build-scripts/20compilation_templates.sh
 clean_compilation_templates:
-	build-scripts/clean_compilation_templates.sh
-
-bootstrap_buildout:
-	build-scripts/bootstrap_buildout.sh
-clean_bootstrap_buildout:
-	build-scripts/clean_bootstrap_buildout.sh
-
-distrib_files:
-	build-scripts/distrib_files.sh
+	build-scripts/clean_20compilation_templates.sh
+
+bootstrap_and_compile: 30bootstrap_and_compile
+30bootstrap_and_compile:
+	build-scripts/30bootstrap_and_compile.sh
+clean_bootstrap_and_compile:
+	build-scripts/clean_30bootstrap_and_compile.sh
+
+backup_tarball: 40backup_tarball
+40backup_tarball:
+	build-scripts/40backup_tarball.sh
+clean_backup_tarball:
+	build-scripts/clean_40backup_tarball.sh
+
+distrib_files: 50distrib_files
+50distrib_files:
+	build-scripts/50distrib_files.sh
 clean_distrib_files:
-	build-scripts/clean_distrib_files.sh
+	build-scripts/clean_50distrib_files.sh
 
-obs:
-	build-scripts/obs.sh
+obs: 60obs
+60obs:
+	build-scripts/60obs.sh
 clean_obs:
-	build-scripts/clean_obs.sh
+	build-scripts/clean_60obs.sh
 
-.PHONY: all clean build clean_build build_tree clean_build_tree compilation_templates clean_compilation_templates bootstrap_buildout clean_bootstrap_buildout distrib_files clean_distrib_files obs clean_obs
+.PHONY: all clean no-obs clean_no-obs build clean_build after_build clean_after_build 10build_tree build_tree clean_build_tree 20compilation_templates compilation_templates clean_compilation_templates 30bootstrap_and_compile bootstrap_and_compile clean_bootstrap_buildout 40backup_tarball backup_tarball clean_backup_tarball 50distrib_files distrib_files clean_distrib_files 60obs obs clean_obs

obs/generic/build-scripts/configuration_information.sh → obs/generic/build-scripts/00env.sh

@@ -33,8 +33,10 @@ TARBALL_DIR="$INITIAL_DIR/tarballs/$SOFTWARE_AND_VERSION"
 
 ### BUILD INFORMATION ###
 # Modify the following variables accordingly
-#SR_PATH=$TARBALL_DIR/software_release/component/$SOFTWARE_NAME/buildout.cfg
-SR_PATH="$TARBALL_DIR/software_release/software/$SOFTWARE_NAME/software.cfg"
+# this is the directory with the buildout files, it can be anything but it usually is a slapos repository
+BUILDOUT_DIR="$TARBALL_DIR/slapos_repository/"
+# this can also be a component rather than a software, replace "software" with "component" and "buildout" to use a component instead
+BUILDOUT_ENTRY_POINT="$BUILDOUT_DIR/software/$SOFTWARE_NAME/software.cfg"
 
 COMPILATION_TEMPLATES_DIR="$INITIAL_DIR/templates/compilation-templates/"
 # BUILD_DIR was formerly BUILD_ROOT_DIRECTORY
@@ -46,17 +48,18 @@ DISTRIB_FILES_DIR="$INITIAL_DIR/distribution-specifics/$SOFTWARE_NAME"
 
 
 ## Path normalization
-INITIAL_DIR="$(realpath -m $INITIAL_DIR)"
-TARGET_DIR="$(realpath -m $TARGET_DIR)"
-OBS_DIR="$(realpath -m $OBS_DIR)"
-DIST_DIR="$(realpath -m $DIST_DIR)"
-TARBALL_DIR="$(realpath -m $TARBALL_DIR)"
-SR_PATH="$(realpath -m $SR_PATH)"
-COMPILATION_TEMPLATES_DIR="$(realpath -m $COMPILATION_TEMPLATES_DIR)"
-BUILD_DIR="$(realpath -m $BUILD_DIR)"
-RUN_BUILDOUT_DIR="$(realpath -m $RUN_BUILDOUT_DIR)"
-DISTRIB_TEMPLATES_DIR="$(realpath -m $DISTRIB_TEMPLATES_DIR)"
-DISTRIB_FILES_DIR="$(realpath -m $DISTRIB_FILES_DIR)"
+INITIAL_DIR=$(realpath -m "$INITIAL_DIR")
+TARGET_DIR=$(realpath -m "$TARGET_DIR")
+OBS_DIR=$(realpath -m "$OBS_DIR")
+DIST_DIR=$(realpath -m "$DIST_DIR")
+TARBALL_DIR=$(realpath -m "$TARBALL_DIR")
+BUILDOUT_DIR=$(realpath -m "$BUILDOUT_DIR")
+BUILDOUT_ENTRY_POINT=$(realpath -m "$BUILDOUT_ENTRY_POINT")
+COMPILATION_TEMPLATES_DIR=$(realpath -m "$COMPILATION_TEMPLATES_DIR")
+BUILD_DIR=$(realpath -m "$BUILD_DIR")
+RUN_BUILDOUT_DIR=$(realpath -m "$RUN_BUILDOUT_DIR")
+DISTRIB_TEMPLATES_DIR=$(realpath -m "$DISTRIB_TEMPLATES_DIR")
+DISTRIB_FILES_DIR=$(realpath -m "$DISTRIB_FILES_DIR")
 
 ## Regular expressions for templates
 NAME_REGEX="s|%SOFTWARE_NAME%|$SOFTWARE_NAME|g;s|%SOFTWARE_AND_VERSION%|$SOFTWARE_AND_VERSION|g"
@@ -68,7 +71,7 @@ VERSION_REGEX="s|%RECIPE_VERSION%|$RECIPE_VERSION|g;s|%SOFTWARE_VERSION%|$SOFTWA
 # Note: %PATCHES_DIRECTORY% not supported yet
 # directories (supporting new macros)
 DIR_REGEX="s|%TARGET_DIR%|$TARGET_DIR|g;s|%BUILD_DIR%|$BUILD_DIR|g;s|%RUN_BUILDOUT_DIR%|$RUN_BUILDOUT_DIR|g"
-PATH_REGEX="s|%SR_PATH%|$SR_PATH|g"
+PATH_REGEX="s|%BUILDOUT_ENTRY_POINT%|$BUILDOUT_ENTRY_POINT|g"
 # directories (supporting legacy macros, not used anymore)
 #OLD_DIR_REGEX="s|\%TARGET_DIRECTORY\%|$TARGET_DIR|g;s|\%BUILD_ROOT_DIRECTORY\%|$BUILD_DIR|g;s|\%BUILD_DIRECTORY\%|$RUN_BUILDOUT_DIR|g"
 # concatenate all regex using ; (quoted, not to end the command)

obs/generic/build-scripts/10build_tree.sh

+#!/bin/bash
+set -e
+
+source build-scripts/00env.sh
+
+cd $INITIAL_DIR
+### Retrieve the buildout directory.
+# Note: This is usually a slapos git repository but it can be otherwise.
+# There needs to be at least one buildout file to point to (the entry point) which can extend other build files. See the buildout documentation if this is unclear.
+mkdir -p $TARBALL_DIR
+#cp -r /home/test/other-projects/model/mca.my-slapos $TARBALL_DIR/software_release
+git clone https://lab.nexedi.com/Ophelie/slapos $BUILDOUT_DIR
+cd $BUILDOUT_DIR
+git checkout metadata-collect-agent
+
+cd $INITIAL_DIR
+### Peparing the build directories
+mkdir -p $RUN_BUILDOUT_DIR/{eggs,extends-cache,download-cache/dist}

obs/generic/build-scripts/compilation_templates.sh → obs/generic/build-scripts/20compilation_templates.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 cd $INITIAL_DIR
 # Prepare the templates in $COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME

obs/generic/build-scripts/30bootstrap_and_compile.sh

+#!/bin/bash
+set -e
+
+source build-scripts/00env.sh
+
+cd $INITIAL_DIR
+### Download the bootstrap script
+mkdir -p $RUN_BUILDOUT_DIR
+cd $RUN_BUILDOUT_DIR
+wget https://bootstrap.pypa.io/bootstrap-buildout.py
+
+cd $INITIAL_DIR
+### Create a $RUN_BUILDOUT_DIR/bin/buildout (bootstraping) and run it (actual compilation).
+# Note: it creates a lot of things in $RUN_BUILDOUT_DIR/eggs/ and uses software_release/ at some point
+mkdir -p $RUN_BUILDOUT_DIR
+cd $RUN_BUILDOUT_DIR
+# should be with gcc here and without in OBS
+cp $COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME/local_buildout.cfg buildout.cfg
+## next line explained
+# 1st cmd: bootstrap buildout (creates bin/buildout)
+# 2nd cmd: backup $RUN_BUILDOUT_DIR/bin/buildout (to be restored for OBS)
+# 3rd cmd: run buildout (which modifies itself via rebootstrapping when compiling python)
+(python2.7 -S bootstrap-buildout.py --buildout-version 2.7.1+slapos016 --setuptools-version 44.1.1 --setuptools-to-dir eggs -f http://www.nexedi.org/static/packages/source/slapos.buildout/ && cp bin/buildout bin/backup.buildout && ./bin/buildout -v)

obs/generic/build-scripts/bootstrap_buildout.sh → obs/generic/build-scripts/40backup_tarball.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 cd $INITIAL_DIR
-# Download the bootstrap script
-mkdir -p $RUN_BUILDOUT_DIR
-cd $RUN_BUILDOUT_DIR
-wget https://bootstrap.pypa.io/bootstrap-buildout.py
+### Fix the go/ directory.
 
-cd $INITIAL_DIR
-# Create a build/bin/buildout (bootstraping) and run it (actual compilation).
-# Note: it creates a lot of things in build/eggs/ and uses software_release/ at some point
-mkdir -p $RUN_BUILDOUT_DIR
-cd $RUN_BUILDOUT_DIR
-# should be with gcc here and without in OBS
-cp $COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME/local_buildout.cfg buildout.cfg
-## next line explained
-# 1st cmd: bootstrap buildout (creates bin/buildout)
-# 2nd cmd: backup bin/buildout (to be restored for OBS)
-# 3rd cmd: run buildout (which apparently modifies itself)
-(python2.7 -S bootstrap-buildout.py --buildout-version 2.7.1+slapos016 --setuptools-version 44.1.1 --setuptools-to-dir eggs -f http://www.nexedi.org/static/packages/source/slapos.buildout/ && cp bin/buildout bin/backup.buildout && ./bin/buildout -v)
-#cp $COMPILATION_TEMPLATES_DIR/tmp/buildout_with_gcc.cfg buildout.cfg
-
-cd $INITIAL_DIR
-# Fix the go/ directory.
-
-# For some reason the user does not have the "write" permission on some directories of go/. As it is
+# For some reason the user does not have the "write" permission on some directories within $RUN_BUILDOUT_DIR/go/. As it is
 # needed on a directory to delete a file in it, the if block adds the permissions to every
 # directories in go/.
 # This is performed before copying the directory tree elsewhere so that every copy is fixed.
 # It also allows the cleaning script to delete the result of the current script.
 if [ -d $RUN_BUILDOUT_DIR/go ]; then
-        find $RUN_BUILDOUT_DIR/go -name "*" -type d -exec chmod u+xw {} +
+	find $RUN_BUILDOUT_DIR/go -name "*" -type d -exec chmod u+xw {} +
 fi
 
 cd $INITIAL_DIR
-# Backup $TARBALL_DIR for debugging or other purpose
+### Backup $TARBALL_DIR for debugging or other purpose
 
 # add "backup." before the directory name pointed to by $TARBALL_DIR
 BACKUP_DIR=$TARBALL_DIR/../backup.$SOFTWARE_AND_VERSION

obs/generic/build-scripts/distrib_files.sh → obs/generic/build-scripts/50distrib_files.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 cd $INITIAL_DIR
 # Prepare the templates in $DISTRIB_TEMPLATES_DIR/tmp/

obs/generic/build-scripts/obs.sh → obs/generic/build-scripts/60obs.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 # temporary directory for the files needed by OBS
 TMP_TARGET=$INITIAL_DIR/obs-tmp/$SOFTWARE_NAME
@@ -31,4 +31,5 @@ cp $TMP_TARGET/debian.tar.gz $OBS_DIR
 cp $TMP_TARGET/*.dsc $OBS_DIR
 
 cd $OBS_DIR
+osc add *.dsc *.tar.gz
 osc commit

obs/generic/build-scripts/build_tree.sh

-#!/bin/bash
-set -e
-
-source build-scripts/configuration_information.sh
-
-cd $INITIAL_DIR
-mkdir -p $TARBALL_DIR
-# Re-downloading the software release to pack
-#cp -r /home/test/other-projects/model/1wip_simplify-fluentbit_my-slapos .
-cp -r /home/test/other-projects/model/mca.my-slapos $TARBALL_DIR/software_release
-#cp -r /home/test/other-projects/new_slapos.package/sid.fluentbit.my-slapos $TARBALL_DIR
-#mv $TARBALL_DIR/mca.my-slapos $TARBALL_DIR/software_release
-
-cd $INITIAL_DIR
-# Peparing the build directories
-mkdir -p $RUN_BUILDOUT_DIR/{eggs,extends-cache,download-cache/dist}

obs/generic/build-scripts/clean_build_tree.sh → obs/generic/build-scripts/clean_10build_tree.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 cd $INITIAL_DIR
 # For some reason the user does not have the "write" permission on some directories of go/. As it is

obs/generic/build-scripts/clean_compilation_templates.sh → obs/generic/build-scripts/clean_20compilation_templates.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 cd $INITIAL_DIR
 # Clean temporary directory for templates

obs/generic/build-scripts/clean_bootstrap_buildout.sh → obs/generic/build-scripts/clean_30bootstrap_and_compile.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 cd $INITIAL_DIR
 # Remove the products of the script bootstrap_buildout.sh

obs/generic/build-scripts/clean_40backup_tarball.sh

+#!/bin/bash
+set -e
+
+source build-scripts/00env.sh
+
+cd $INITIAL_DIR
+# Nothing is done here, as 04backup_tarball.sh only modifies permissions in $RUN_BUILDOUT_DIR/go/ and backs up the $TARBALL_DIR (one do not want to delete the backup if it's not to create a new one).

obs/generic/build-scripts/clean_distrib_files.sh → obs/generic/build-scripts/clean_50distrib_files.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 cd $INITIAL_DIR
 # Clean the temporary directory of distribution files for $SOFTWARE_NAME

obs/generic/build-scripts/clean_obs.sh → obs/generic/build-scripts/clean_60obs.sh

 #!/bin/bash
 set -e
 
-source build-scripts/configuration_information.sh
+source build-scripts/00env.sh
 
 # reverse the action of obsoleting the potentially altered buildout and restoring the backed up one
 if [ ! -e $RUN_BUILDOUT_DIR/bin/backup.buildout ]; then

obs/generic/templates/compilation-templates/_generic/local_buildout.cfg.in

@@ -3,7 +3,7 @@ rootdir = %TARGET_DIR%
 destdir = %BUILD_DIR%
 builddir = %RUN_BUILDOUT_DIR%
 extends =
-  %SR_PATH%
+  %BUILDOUT_ENTRY_POINT%
 versions = versions
 extends-cache = extends-cache
 download-cache = download-cache

obs/generic/templates/compilation-templates/dep--fluent-bit/local_buildout.cfg.in

@@ -3,7 +3,7 @@ rootdir = %TARGET_DIR%
 destdir = %BUILD_DIR%
 builddir = %RUN_BUILDOUT_DIR%
 extends =
-  %SR_PATH%
+  %BUILDOUT_ENTRY_POINT%
 versions = versions
 extends-cache = extends-cache
 download-cache = download-cache

obs/generic/templates/compilation-templates/dep--mca--static/local_buildout.cfg.in

@@ -3,7 +3,7 @@ rootdir = %TARGET_DIR%
 destdir = %BUILD_DIR%
 builddir = %RUN_BUILDOUT_DIR%
 extends =
-  %SR_PATH%
+  %BUILDOUT_ENTRY_POINT%
 versions = versions
 extends-cache = extends-cache
 download-cache = download-cache

obs/generic/templates/compilation-templates/fluent-bit/local_buildout.cfg.in

@@ -3,7 +3,7 @@ rootdir = %TARGET_DIR%
 destdir = %BUILD_DIR%
 builddir = %RUN_BUILDOUT_DIR%
 extends =
-  %SR_PATH%
+  %BUILDOUT_ENTRY_POINT%
 versions = versions
 extends-cache = extends-cache
 download-cache = download-cache

obs/generic/templates/compilation-templates/mca/local_buildout.cfg.in

@@ -3,7 +3,7 @@ rootdir = %TARGET_DIR%
 destdir = %BUILD_DIR%
 builddir = %RUN_BUILDOUT_DIR%
 extends =
-  %SR_PATH%
+  %BUILDOUT_ENTRY_POINT%
 versions = versions
 extends-cache = extends-cache
 download-cache = download-cache