obs/generic: Support custom compilation template files. Add some fixes and tweaks.

Note: Allow to use custom distribution files instead of the template-generated ones.

obs/generic/build-scripts/bootstrap_buildout.sh

@@ -15,7 +15,7 @@ cd $INITIAL_DIR
 mkdir -p $RUN_BUILDOUT_DIR
 cd $RUN_BUILDOUT_DIR
 # should be with gcc here and without in OBS
-cp $COMPILATION_TEMPLATES_DIR/tmp/buildout_without_gcc.cfg buildout.cfg
+cp $COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME/local_buildout.cfg buildout.cfg
 ## next line explained
 # 1st cmd: bootstrap buildout (creates bin/buildout)
 # 2nd cmd: backup bin/buildout (to be restored for OBS)

obs/generic/build-scripts/clean_compilation_templates.sh

@@ -5,4 +5,4 @@ source build-scripts/configuration_information.sh
 
 cd $INITIAL_DIR
 # Clean temporary directory for templates
-rm -rf $COMPILATION_TEMPLATES_DIR/tmp/
+rm -rf $COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME

obs/generic/build-scripts/clean_distrib_files.sh

@@ -4,6 +4,10 @@ set -e
 source build-scripts/configuration_information.sh
 
 cd $INITIAL_DIR
-# Clean temporary directory for templates
-rm -rf $DISTRIB_TEMPLATES_DIR/tmp/
-rm -rf $DISTRIB_FILES_DIR
+# Clean the temporary directory of distribution files for $SOFTWARE_NAME
+rm -rf $DISTRIB_TEMPLATES_DIR/tmp/$SOFTWARE_NAME
+# Only delete the directory is the file .toclean-stam is present in it. It should only be created
+# when the directory is automatically generated so that the cleaning script can reverse the action.
+if [ -e $DISTRIB_FILES_DIR/.toclean-stamp ]; then
+	rm -rf $DISTRIB_FILES_DIR
+fi

obs/generic/build-scripts/clean_obs.sh

@@ -3,6 +3,16 @@ set -e
 
 source build-scripts/configuration_information.sh
 
-TMP_DIR=$INITIAL_DIR/tmp/
-TMP_DIR=`realpath -m $TMP_DIR`
-rm -rf $TMP_DIR
+# reverse the action of obsoleting the potentially altered buildout and restoring the backed up one
+if [ ! -e $RUN_BUILDOUT_DIR/bin/backup.buildout ]; then
+	if [ -e $RUN_BUILDOUT_DIR/bin/buildout ]; then
+		cp $RUN_BUILDOUT_DIR/bin/buildout $RUN_BUILDOUT_DIR/bin/backup.buildout
+	fi
+fi
+if [ -e $RUN_BUILDOUT_DIR/bin/old.buildout ]; then
+	mv $RUN_BUILDOUT_DIR/bin/old.buildout $RUN_BUILDOUT_DIR/bin/buildout
+fi
+
+TMP_TARGET=$INITIAL_DIR/obs-tmp/$SOFTWARE_NAME
+TMP_TARGET=`realpath -m $TMP_TARGET`
+rm -rf $TMP_TARGET

obs/generic/build-scripts/compilation_templates.sh

@@ -4,8 +4,15 @@ set -e
 source build-scripts/configuration_information.sh
 
 cd $INITIAL_DIR
-# Prepare the templates in $COMPILATION_TEMPLATES_DIR/tmp/
-mkdir -p $COMPILATION_TEMPLATES_DIR/tmp/
-sed $ALL_REGEX $COMPILATION_TEMPLATES_DIR/buildout_with_gcc.cfg.in > 	$COMPILATION_TEMPLATES_DIR/tmp/buildout_with_gcc.cfg
-sed $ALL_REGEX $COMPILATION_TEMPLATES_DIR/buildout_without_gcc.cfg.in > 	$COMPILATION_TEMPLATES_DIR/tmp/buildout_without_gcc.cfg
-sed $ALL_REGEX $COMPILATION_TEMPLATES_DIR/Makefile.in > 			$COMPILATION_TEMPLATES_DIR/tmp/Makefile
+# Prepare the templates in $COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME
+mkdir -p $COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME
+
+# if a custom directory exist for compilation templates, choose it
+# otherwise, default to .../_generic
+ACTUAL_TEMPLATES_DIR=$COMPILATION_TEMPLATES_DIR/$SOFTWARE_NAME
+if [ ! -d $ACTUAL_TEMPLATES_DIR ]; then
+	ACTUAL_TEMPLATES_DIR=$COMPILATION_TEMPLATES_DIR/_generic
+fi
+
+sed $ALL_REGEX $ACTUAL_TEMPLATES_DIR/local_buildout.cfg.in > 	$COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME/local_buildout.cfg
+sed $ALL_REGEX $ACTUAL_TEMPLATES_DIR/Makefile.in > 			$COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME/Makefile

obs/generic/build-scripts/configuration_information.sh

@@ -33,12 +33,12 @@ TARBALL_DIR=$INITIAL_DIR/tarballs/$SOFTWARE_AND_VERSION
 #SR_PATH=$TARBALL_DIR/software_release/component/$SOFTWARE_NAME/buildout.cfg
 SR_PATH=$TARBALL_DIR/software_release/software/$SOFTWARE_NAME/software.cfg
 
-COMPILATION_TEMPLATES_DIR=$INITIAL_DIR/templates/compilation-templates
+COMPILATION_TEMPLATES_DIR=$INITIAL_DIR/templates/compilation-templates/
 # BUILD_DIR was formerly BUILD_ROOT_DIRECTORY
 BUILD_DIR=$TARBALL_DIR/build/
 # RUN_BUILDOUT_DIR is where the parts/ directory will be ; formerly BUILD_DIRECTORY
 RUN_BUILDOUT_DIR=$BUILD_DIR/$TARGET_DIR
-DISTRIB_TEMPLATES_DIR=$INITIAL_DIR/templates/default-distribution-templates
+DISTRIB_TEMPLATES_DIR=$INITIAL_DIR/templates/distribution-templates
 DISTRIB_FILES_DIR=$INITIAL_DIR/distribution-specifics/$SOFTWARE_NAME
 
 
@@ -56,7 +56,7 @@ DISTRIB_TEMPLATES_DIR=`realpath -m $DISTRIB_TEMPLATES_DIR`
 DISTRIB_FILES_DIR=`realpath -m $DISTRIB_FILES_DIR`
 
 ## Regular expressions for templates
-NAME_REGEX="s|%SOFTWARE_NAME%|$SOFTWARE_NAME|g"
+NAME_REGEX="s|%SOFTWARE_NAME%|$SOFTWARE_NAME|g;s|%SOFTWARE_AND_VERSION%|$SOFTWARE_AND_VERSION|g"
 # versions (not used at the moment)
 # versions (supporting legact macros)
 #OLD_VERSION_REGEX="s|\%RECIPE_VERSION\%|$RECIPE_VERSION|g;s|\%VERSION\%|$SOFTWARE_VERSION|g;s|\%DEBIAN_REVISION\%|$DEBIAN_REVISION|g;s|\%COMPOUND_VERSION\%|$COMPOUND_VERSION|g"

obs/generic/build-scripts/distrib_files.sh

@@ -5,16 +5,26 @@ source build-scripts/configuration_information.sh
 
 cd $INITIAL_DIR
 # Prepare the templates in $DISTRIB_TEMPLATES_DIR/tmp/
-mkdir -p $DISTRIB_TEMPLATES_DIR/tmp/debian/
-sed $ALL_REGEX $DISTRIB_TEMPLATES_DIR/default.dsc.in >			$DISTRIB_TEMPLATES_DIR/tmp/$SOFTWARE_AND_VERSION.dsc
+TMP_TARGET=$DISTRIB_TEMPLATES_DIR/tmp/$SOFTWARE_NAME/
+mkdir -p $TMP_TARGET
+sed $ALL_REGEX $DISTRIB_TEMPLATES_DIR/default.dsc.in >			$TMP_TARGET/$SOFTWARE_AND_VERSION.dsc
 # debian directory
-sed $ALL_REGEX $DISTRIB_TEMPLATES_DIR/debian_default/changelog.in >	$DISTRIB_TEMPLATES_DIR/tmp/debian/changelog
-sed $ALL_REGEX $DISTRIB_TEMPLATES_DIR/debian_default/control.in >	$DISTRIB_TEMPLATES_DIR/tmp/debian/control
-sed $ALL_REGEX $DISTRIB_TEMPLATES_DIR/debian_default/dirs.in >		$DISTRIB_TEMPLATES_DIR/tmp/debian/dirs
-cp -r $DISTRIB_TEMPLATES_DIR/debian_default/{compat,copyright,rules,source} $DISTRIB_TEMPLATES_DIR/tmp/debian/
+mkdir -p $TMP_TARGET/debian/
+sed $ALL_REGEX $DISTRIB_TEMPLATES_DIR/debian_default/changelog.in >	$TMP_TARGET/debian/changelog
+sed $ALL_REGEX $DISTRIB_TEMPLATES_DIR/debian_default/control.in >	$TMP_TARGET/debian/control
+sed $ALL_REGEX $DISTRIB_TEMPLATES_DIR/debian_default/dirs.in >		$TMP_TARGET/debian/dirs
+cp -r $DISTRIB_TEMPLATES_DIR/debian_default/{compat,copyright,rules,source} $TMP_TARGET/debian/
 
 # TODO: do it with "find" instead?
-#find $DISTRIB_TEMPLATES_DIR/debian_defaults/ -type f -name * -exec sed $ALL_REGEX {} + > $DISTRIB_TEMPLATES_DIR/tmp/debian/
+#find $DISTRIB_TEMPLATES_DIR/debian_defaults/ -type f -name * -exec sed $ALL_REGEX {} + > $TMP_TARGET/debian/
 
-mkdir -p $INITIAL_DIR/distribution-specifics/$SOFTWARE_NAME/
-cp -r $DISTRIB_TEMPLATES_DIR/tmp/* $DISTRIB_FILES_DIR/
+# If a directory for the current software does not already exist in distrubion-specifics:
+# create it and copy the product of the templates in it,
+# otherwise do nothing.
+# The files created form the templates are still available in $TMP_TARGET
+if [ ! -d $DISTRIB_FILES_DIR ]; then
+	mkdir -p $DISTRIB_FILES_DIR
+	cp -r $TMP_TARGET/* $DISTRIB_FILES_DIR/
+	# A file is created so that the cleaning script can reverse the action of the current script.
+	echo -e "This file's only purpose is to tell the cleaning script to delete this directory.\nThere should not be such file in a manually crafted directory.\n" >> $DISTRIB_FILES_DIR/.toclean-stamp
+fi

obs/generic/build-scripts/obs.sh

@@ -4,14 +4,14 @@ set -e
 source build-scripts/configuration_information.sh
 
 # temporary directory for the files needed by OBS
-TMP_DIR=$INITIAL_DIR/tmp/
-TMP_DIR=`realpath -m $TMP_DIR`
-echo TMP_DIR = $TMP_DIR
+TMP_TARGET=$INITIAL_DIR/obs-tmp/$SOFTWARE_NAME
+TMP_TARGET=`realpath -m $TMP_TARGET`
+echo TMP_TARGET = $TMP_TARGET
 
-# copy the Makefile at the root of the tarball
-cp $COMPILATION_TEMPLATES_DIR/tmp/Makefile $TARBALL_DIR/Makefile
+# copy the generated Makefile at the root of the tarball
+cp $COMPILATION_TEMPLATES_DIR/tmp/$SOFTWARE_NAME/Makefile $TARBALL_DIR/Makefile
 # save the local TARBALL_DIR to replace it with the TARBALL_DIR of OBS' VM
-echo $TARBALL_DIR > $TARBALL_DIR/cache_creation_build_directory
+echo $TARBALL_DIR > $TARBALL_DIR/local_build_directory
 # restore bin/buildout
 # note: when installing python, buildout "rebootstrap" itself to use the installed python: it would fail on OBS' VM
 mv $RUN_BUILDOUT_DIR/bin/backup.buildout $RUN_BUILDOUT_DIR/bin/buildout
@@ -20,15 +20,15 @@ mv $RUN_BUILDOUT_DIR/bin/backup.buildout $RUN_BUILDOUT_DIR/bin/buildout
 rm -rf $RUN_BUILDOUT_DIR/{.installed.cfg,parts/}
 
 ## prepare the files for OBS
-mkdir -p $TMP_DIR
+mkdir -p $TMP_TARGET
 # -C option allows to give tar an absolute path without archiving the directory from / (i.e. home/user/[...])
-tar czf $TMP_DIR/${SOFTWARE_AND_VERSION}${ARCHIVE_EXT} -C $INITIAL_DIR/tarballs/ $SOFTWARE_AND_VERSION/
-tar czf $TMP_DIR/debian.tar.gz -C $DIST_DIR/ debian/
-cp $DIST_DIR/*.dsc $TMP_DIR/
+tar czf $TMP_TARGET/${SOFTWARE_AND_VERSION}${ARCHIVE_EXT} -C $INITIAL_DIR/tarballs/ $SOFTWARE_AND_VERSION/
+tar czf $TMP_TARGET/debian.tar.gz -C $DIST_DIR/ debian/
+cp $DIST_DIR/*.dsc $TMP_TARGET/
 # move the files for OBS
-cp $TMP_DIR/${SOFTWARE_AND_VERSION}${ARCHIVE_EXT} $OBS_DIR
-cp $TMP_DIR/debian.tar.gz $OBS_DIR
-cp $TMP_DIR/*.dsc $OBS_DIR
+cp $TMP_TARGET/${SOFTWARE_AND_VERSION}${ARCHIVE_EXT} $OBS_DIR
+cp $TMP_TARGET/debian.tar.gz $OBS_DIR
+cp $TMP_TARGET/*.dsc $OBS_DIR
 
 cd $OBS_DIR
 osc commit

obs/generic/distribution-specifics/example/.toclean-stamp

+This file's only purpose is to tell the cleaning script to delete this directory.
+There should not be such file in a manually crafted directory.
+

obs/generic/templates/compilation-templates/Makefile.in → obs/generic/templates/compilation-templates/_generic/Makefile.in

@@ -13,7 +13,7 @@ INSTALL_DIR = $(DESTDIR)$(TARGET_DIR)
 RUN_BUILDOUT_DIR = $(BUILD_DIR)$(TARGET_DIR)
 #PATCHES_DIR := $(shell pwd)/patches
 # get the path of the BUILD_DIR of the first build (performed to prepare the cache for OBS)
-OLD_TARBALL_DIR := $(shell cat cache_creation_build_directory)
+OLD_TARBALL_DIR := $(shell cat local_build_directory)
 
 #Use to get path of buildout correct
 #ORIGINAL_DIRECTORY := $(shell cat ./original_directory)

obs/generic/templates/compilation-templates/buildout_without_gcc.cfg.in → obs/generic/templates/compilation-templates/_generic/local_buildout.cfg.in

@@ -4,10 +4,6 @@ destdir = %BUILD_DIR%
 builddir = %RUN_BUILDOUT_DIR%
 extends =
   %SR_PATH%
-#  %BUILD_DIR%/../software_release/software/fluent-bit/software.cfg
-#  %TARBALL_DIR%/software_release/software/fluent-bit/software.cfg
-#  %TARBALL_DIR%/software_release/component/fluent-bit/buildout.cfg
-#  %BUILD_DIR%/../software_release/component/fluent-bit/buildout.cfg
 versions = versions
 extends-cache = extends-cache
 download-cache = download-cache

obs/generic/templates/compilation-templates/fluent-bit/Makefile.in

+
+PY = $(PYTHON)
+#INSTALL_DIR = $(DESTDIR)/opt/test
+#TARGET_DIR = /opt/slapos
+# This templates are replaced by build-scripts/template_stage.sh
+# according to the values and regular expressions defined in
+# build-scripts/configuration_information.sh
+# The variable TARGET_DIR is pretty useless atm
+TARGET_DIR = %TARGET_DIR%
+TARBALL_DIR = $(shell pwd)
+BUILD_DIR = $(TARBALL_DIR)/build
+INSTALL_DIR = $(DESTDIR)$(TARGET_DIR)
+RUN_BUILDOUT_DIR = $(BUILD_DIR)$(TARGET_DIR)
+#PATCHES_DIR := $(shell pwd)/patches
+# get the path of the BUILD_DIR of the first build (performed to prepare the cache for OBS)
+OLD_TARBALL_DIR := $(shell cat local_build_directory)
+
+#Use to get path of buildout correct
+#ORIGINAL_DIRECTORY := $(shell cat ./original_directory)
+
+all: build
+
+build: build-stamp
+build-stamp:
+	@echo "Fixing buildout path to $(TARBALL_DIR) rather than $(OLD_TARBALL_DIR) for buildout"
+################################################################################ 
+#	grep -rIl '$(OLD_DIRECTORY)' $(BUILD_DIR) 2> /dev/null | \
+#		xargs sed -i 's#$(OLD_DIRECTORY)#$(BUILD_DIR)#g' || \
+#		echo "No path to fix."
+################################################################################ 
+	ls -l $(RUN_BUILDOUT_DIR) # DEBUG
+	ls -l $(RUN_BUILDOUT_DIR)/bin/ # DEBUG
+	cd $(RUN_BUILDOUT_DIR); sed -i 's#$(OLD_TARBALL_DIR)#$(TARBALL_DIR)#g' buildout.cfg bin/*
+	echo RUN_BUILDOUT_DIR = $(RUN_BUILDOUT_DIR) \; PY = $(PY)
+	cd $(RUN_BUILDOUT_DIR) && \
+		$(PY) ./bin/buildout -v
+
+	@touch build-stamp
+
+clean:
+	# TODO: implement a proper cleaning
+	#rm -rf $(BUILD_DIR)
+	rm -f *-stamp
+
+install: all
+	#cd slapos; make install
+	mkdir -p $(INSTALL_DIR)/bin
+	mkdir -p $(INSTALL_DIR)/share/
+	#cp `find $(RUN_BUILDOUT_DIR) | grep -e "^$(RUN_BUILDOUT_DIR)/[^/][^/]*/bin/*` $(DESTDIR)/usr/bin/
+	-cp -r $(RUN_BUILDOUT_DIR)/parts/fluent-bit/bin/* $(INSTALL_DIR)/bin/
+	-cp -r $(RUN_BUILDOUT_DIR)/parts/fluent-bit/share/* $(INSTALL_DIR)/share/
+	ls -Rl $(INSTALL_DIR)/bin/ # DEBUG
+
+.PHONY: build all clean install

obs/generic/templates/compilation-templates/buildout_with_gcc.cfg.in → obs/generic/templates/compilation-templates/fluent-bit/local_buildout.cfg.in

@@ -4,18 +4,10 @@ destdir = %BUILD_DIR%
 builddir = %RUN_BUILDOUT_DIR%
 extends =
   %SR_PATH%
-#  %BUILD_DIR%/../software_release/software/fluent-bit/software.cfg
-#  %TARBALL_DIR%/software_release/software/fluent-bit/software.cfg
-#  %TARBALL_DIR%/software_release/component/fluent-bit/buildout.cfg
-#  %BUILD_DIR%/../software_release/component/fluent-bit/buildout.cfg
 versions = versions
 extends-cache = extends-cache
 download-cache = download-cache
 
-[gcc]
-# force usage of gcc from slapos to have it in the cache
-max_version = 0
-
 [versions]
 setuptools = 44.1.1
 # Use SlapOS patched zc.buildout

obs/generic/templates/compilation-templates/mca/Makefile.in

+
+PY = $(PYTHON)
+#INSTALL_DIR = $(DESTDIR)/opt/test
+#TARGET_DIR = /opt/slapos
+# This templates are replaced by build-scripts/template_stage.sh
+# according to the values and regular expressions defined in
+# build-scripts/configuration_information.sh
+# The variable TARGET_DIR is pretty useless atm
+TARGET_DIR = %TARGET_DIR%
+TARBALL_DIR = $(shell pwd)
+BUILD_DIR = $(TARBALL_DIR)/build
+INSTALL_DIR = $(DESTDIR)$(TARGET_DIR)
+RUN_BUILDOUT_DIR = $(BUILD_DIR)$(TARGET_DIR)
+#PATCHES_DIR := $(shell pwd)/patches
+# get the path of the BUILD_DIR of the first build (performed to prepare the cache for OBS)
+OLD_TARBALL_DIR := $(shell cat local_build_directory)
+
+#Use to get path of buildout correct
+#ORIGINAL_DIRECTORY := $(shell cat ./original_directory)
+
+all: build
+
+build: build-stamp
+build-stamp:
+	@echo "Fixing buildout path to $(TARBALL_DIR) rather than $(OLD_TARBALL_DIR) for buildout"
+################################################################################ 
+#	grep -rIl '$(OLD_DIRECTORY)' $(BUILD_DIR) 2> /dev/null | \
+#		xargs sed -i 's#$(OLD_DIRECTORY)#$(BUILD_DIR)#g' || \
+#		echo "No path to fix."
+################################################################################ 
+	ls -l $(RUN_BUILDOUT_DIR) # DEBUG
+	ls -l $(RUN_BUILDOUT_DIR)/bin/ # DEBUG
+	cd $(RUN_BUILDOUT_DIR); sed -i 's#$(OLD_TARBALL_DIR)#$(TARBALL_DIR)#g' buildout.cfg bin/*
+	echo RUN_BUILDOUT_DIR = $(RUN_BUILDOUT_DIR) \; PY = $(PY)
+	cd $(RUN_BUILDOUT_DIR) && \
+		$(PY) ./bin/buildout -v
+
+	@touch build-stamp
+
+clean:
+	# TODO: implement a proper cleaning
+	#rm -rf $(BUILD_DIR)
+	rm -f *-stamp
+
+install: all
+	#cd slapos; make install
+	mkdir -p $(INSTALL_DIR)/bin
+	mkdir -p $(INSTALL_DIR)/etc/
+	mkdir -p $(INSTALL_DIR)/lib/
+	mkdir -p $(INSTALL_DIR)/share/
+	#cp `find $(RUN_BUILDOUT_DIR) | grep -e "^$(RUN_BUILDOUT_DIR)/[^/][^/]*/bin/*` $(DESTDIR)/usr/bin/
+	-cp -r $(RUN_BUILDOUT_DIR)/parts/mca/bin/* $(INSTALL_DIR)/bin/
+	-cp -r $(RUN_BUILDOUT_DIR)/parts/mca/etc/* $(INSTALL_DIR)/etc/
+	-cp -r $(RUN_BUILDOUT_DIR)/parts/{mca,openssl}/lib/* $(INSTALL_DIR)/lib/
+	-cp -r $(RUN_BUILDOUT_DIR)/parts/{mca,openssl}/share/* $(INSTALL_DIR)/share/
+	ls -Rl $(INSTALL_DIR)/bin/ # DEBUG
+
+.PHONY: build all clean install

obs/generic/templates/compilation-templates/mca/local_buildout.cfg.in

+[buildout]
+rootdir = %TARGET_DIR%
+destdir = %BUILD_DIR%
+builddir = %RUN_BUILDOUT_DIR%
+extends =
+  %SR_PATH%
+versions = versions
+extends-cache = extends-cache
+download-cache = download-cache
+
+[versions]
+setuptools = 44.1.1
+# Use SlapOS patched zc.buildout
+zc.buildout = 2.7.1+slapos016
+# Use SlapOS patched zc.recipe.egg (zc.recipe.egg 2.x is for Buildout 2)
+zc.recipe.egg = 2.0.3+slapos003

obs/generic/templates/default-distribution-templates/debian_default/dirs.in

-/opt/%SOFTWARE_NAME%/bin/
-/opt/%SOFTWARE_NAME%/lib/
-/opt/%SOFTWARE_NAME%/etc/

obs/generic/templates/default-distribution-templates/debian_default/control.in → obs/generic/templates/distribution-templates/debian_default/control.in

@@ -6,7 +6,7 @@ Build-Depends: debhelper,
                chrpath,
                python
 
-Package: mca
+Package: %SOFTWARE_NAME%
 Architecture: any
 Depends: ${misc:Depends},
          ${shlibs:Depends},

obs/generic/templates/distribution-templates/debian_default/dirs.in

+/opt/%SOFTWARE_AND_VERSION%/bin/
+/opt/%SOFTWARE_AND_VERSION%/lib/
+/opt/%SOFTWARE_AND_VERSION%/etc/