re6st: switch to Python 3

Co-authored-by: Julien Muchembled <jm@nexedi.com>

obs/re6st/Makefile.in

-# Note that this file and related scripts (bootstrap, install-eggs) don't
-# contain anything specific to re6stnet. They could be reused as is for SlapOS.
+# Note that this file and related scripts (build_python3_if_needed,
+# install-eggs) don't contain anything specific to re6stnet. They could be
+# reused as is for SlapOS.
 
 # We don't strip ELF files because all dists automatically do it.
 # For those that produce debug packages, these packages are anyway quite small
@@ -8,7 +9,6 @@
 BUILD = $(TARGET)
 
 PATH := $(CURDIR)/$(BUILD)/parts/chrpath/bin:$(CURDIR)/$(BUILD)/parts/file/bin:$(CURDIR)/$(BUILD)/parts/perl/bin:$(PATH)
-PYTHON = $(or $(shell command -v python2 || command -v python || command -v python3),$(error no Python found))
 
 all: $(BUILD)/.installed.cfg
 ifneq ($(wildcard upstream.mk),)
@@ -21,11 +21,9 @@ else
 install: _install
 endif
 
-$(BUILD)/bin/buildout:
-	cd $(BUILD) && $(PYTHON) -S $(CURDIR)/bootstrap
-
-$(BUILD)/.installed.cfg: $(BUILD)/bin/buildout $(BUILD)/buildout.cfg
-	cd $(BUILD) && bin/buildout babeld-repository:recipe=
+$(BUILD)/.installed.cfg: $(BUILD)/buildout.cfg
+	cd $(BUILD) && $(shell ./build_python3_if_needed) bin/buildout \
+	buildout:relative-paths=false babeld-repository:recipe=
 	touch $@
 
 PROGS = $(patsubst %,$(DESTDIR)/usr/bin/%,$(BIN))
@@ -39,7 +37,8 @@ $(PROGS): $(BUILD)/.installed.cfg
 
 $(DESTDIR)/$(TARGET): $(BUILD)/.installed.cfg
 	rm -rf $@ && mkdir -p $@/parts
-	cd $(BUILD) && $(PYTHON) $(CURDIR)/install-eggs $@ $(BIN)
+	cd $(BUILD) && parts/python3/bin/python3 \
+	$(CURDIR)/install-eggs $@ $(BIN)
 	for x in $(filter-out $(NOPART),$(shell cd $(BUILD)/parts && echo *)); \
 	do cp --preserve=links -r $(BUILD)/parts/$$x $@/parts; done
 	cd $@ && $(CURDIR)/cleanup && chmod -R u+w .
@@ -63,6 +62,12 @@ $(DESTDIR)/$(TARGET): $(BUILD)/.installed.cfg
 		rm $$d; ln -s /$(TARGET)$$s $$d; \
 	done
 
+uniq = $(if $1,$(firstword $1) $(call uniq,$(filter-out $(firstword $1),$1)))
+_dir = $(patsubst %/,%,$(call uniq,$(filter-out ./, $(dir $1))))
+_anc = $(if $1,$1 $(call _anc,$(call _dir,$1)))
+
 clean:
-	find $(BUILD) -mindepth 1 -maxdepth 1 \
-		$(patsubst %,! -name %,$(BUILD_KEEP)) |xargs rm -rf
+	cd $(BUILD) && find -mindepth 1 \( \
+		$(patsubst %, -path ./% -o,$(CLEAN_KEEP)) \
+		$(patsubst %, ! -path ./%,$(call _anc,$(call _dir,$(CLEAN_KEEP)))) \
+		-print0 \) -prune |xargs -r0 rm -r

obs/re6st/PKGBUILD.in

-# Maintainer: Rafael Monnerat <rafael@nexedi.com>
-pkgname=re6st-node
+pkgname=%PACKAGE%
 pkgver=%VERSION%
 pkgdesc="resilient, scalable, IPv6 network application"
 pkgrel=1
 arch=('x86_64' 'i686')
 license=('GPL')
 depends=(iptables iproute2)
-makedepends=(python2)
+makedepends=(python3)
 install='re6stnet.install'
 source=(${pkgname}_${pkgver}.tar.gz)
 

obs/re6st/bootstrap

-#!/usr/bin/python3 -S
-import glob, os, shutil, sys, tarfile, tempfile, zipfile
-dist = "download-cache/dist"
-tmp = tempfile.mkdtemp()
-try:
-    setuptools, = glob.glob(dist + "/setuptools-*")
-    if setuptools.endswith(".zip"):
-        zipfile.ZipFile(setuptools).extractall(tmp)
-    else:
-        tarfile.TarFile.open(setuptools, "r:*").extractall(tmp)
-    x, = os.listdir(tmp)
-    sys.path.insert(0, os.path.join(tmp, x))
-    from setuptools.command.easy_install import main
-    for x in "bin", "eggs":
-        try:
-            os.mkdir(x)
-        except OSError:
-            pass
-    main(["-f", dist, "-mxd", x, setuptools, "zc.buildout"])
-finally:
-  shutil.rmtree(tmp)
-
-with os.fdopen(os.open("bin/buildout", os.O_CREAT|os.O_WRONLY|os.O_TRUNC, 0o777), 'w') as f:
-    f.write("""\
-#!%s -S
-import os, sys
-d = os.path.dirname(os.path.abspath(os.path.realpath(__file__)))
-d = os.path.dirname(d) + %r
-sys.path[:0] = (%s)
-from zc.buildout.buildout import main
-sys.exit(main())
-""" % (
-    sys.executable,
-    "/%s/" % x,
-    ", ".join(map("d + %r".__mod__, os.listdir(x))),
-    ))

obs/re6st/build_python3_if_needed

+#!/bin/sh
+
+set -e
+
+# This script compiles python3 if current python3 is older than python3.7 (minimum version required by buildout)
+# It returns the path of the python3 executable to use
+
+python3 -c 'import sys; sys.exit(1) if sys.version_info < (3, 7) else print(sys.executable)' && exit
+
+build_dir=.build
+ark=$PWD/opt/re6st/python3.tar.xz
+mkdir -p $build_dir
+cd $build_dir
+if [ ! -x python ]; then
+  tar --strip-components=1 -xaf "$ark"
+  ./configure
+  make -j4
+fi >/dev/null
+
+readlink -f python

obs/re6st/buildout.cfg.in

@@ -10,10 +10,12 @@ download-cache = download-cache
 develop =
 extensions = slapos.rebootstrap
 newest = false
+relative-paths = true
 
-allow-picked-versions = true
+find-links += https://softinst171429.host.vifib.net/public/
+allow-hosts += softinst171429.host.vifib.net
 
-parts += chrpath python-bootstrap
+parts = re6st-ovpn chrpath
 
 [babeld-repository]
 git-executable =
@@ -23,12 +25,27 @@ location = ${buildout:directory}/babeld
 post-install = make clean; make version.h
 environment =
 
-[python-bootstrap]
-recipe = zc.recipe.egg
-interpreter = python.tmp
-eggs = slapos.libnetworkcache
-# https://lab.nexedi.com/nexedi/slapos.buildout/merge_requests/11
-scripts = dummy
+# Adjust interpreter of OpenVPN hooks.
+[re6st-ovpn]
+=> re6stnet
+recipe = slapos.recipe.build
+update =
+  import os, shutil, sys
+  shebang = f'#!{sys.executable} -S\n'
+  ws = self.buildout['re6stnet'].recipe.working_set()[1]
+  for root, dirs, files in os.walk(ws.by_key['re6stnet'].location):
+    for x in files:
+      if x.startswith('ovpn-'):
+        x = os.path.join(root, x)
+        with open(x) as src:
+          if src.readline() != shebang:
+            y = x + '.new'
+            with open(y, 'w') as dst:
+              src.readline()
+              dst.write(shebang)
+              shutil.copyfileobj(src, dst)
+            shutil.copymode(x, y)
+            os.rename(y, x)
 
 # Uguu, upstream buildout.cfg must be patched as it works the other way
 # around from a packager point of view at least, thus at the end static
@@ -38,13 +55,21 @@ scripts = dummy
 # ./configure --prefix=BUILD_DIRECTORY && make install
 # Instead of:
 # ./configure --prefix=INSTALL_DIRECTORY && make install DESTDIR=BUILD_DIRECTORY
-[python2.7]
+[python3-common]
 configure-options +=
   --prefix=%(TARGET)s/parts/${:_buildout_section_name_}
 environment +=
   DESTDIR=%(ROOT)s
+post-install =
+  set `echo -n ${:url} |md5sum`
+  ln -frs ${buildout:download-cache}/$1 ${buildout:directory}/python3.tar.xz
 
-[openssl]
+[python]
+# Do not rely on existing installation of what we are building.
+update +=
+  os.environ['SSL_CERT_DIR'] = self.buildout['openssl']['certs']
+
+[openssl-common]
 shared = false
 prefix = %(TARGET)s/parts/${:_buildout_section_name_}
 make-options +=
@@ -60,3 +85,7 @@ environment =
 # For old GCC like 4.8.5 on SLE 12 SP5.
 environment +=
   CFLAGS=-std=c99 -g -O2
+
+[versions]
+zc.buildout = 3.0.1+slapos005.mr36v6
+re6stnet = %(RE6STNET_VERSION)s:whl

obs/re6st/debian/compat

+10

obs/re6st/debian/prerm

@@ -2,13 +2,12 @@
 set -e
 
 pyclean() {
-  local x IFS='
+  local IFS='
 '
-  x=`dpkg -L $1`
-  rm -f `for x in $x; do
-    case $x in *.py) echo $x[co];; esac
-  done`
+  find `dpkg -L $1 |sed -n 's,[^/]*\.py$,__pycache__,p' |
+    sort -u` -delete 2>/dev/null
 }
+
 pyclean $DPKG_MAINTSCRIPT_PACKAGE
 
 #DEBHELPER#

obs/re6st/debian/rules

@@ -8,6 +8,7 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=-format
 include debian/common.mk
 
 override_dh_auto_install:
+	make DESTDIR=$(TMP) PREFIX=/usr install
 
 override_dh_link:
 

obs/re6st/install-eggs

-#!/usr/bin/python
-import errno, os, shutil, sys
+#!/usr/bin/python3
+import os, shutil, sys
 
 path = set()
 
@@ -17,7 +17,7 @@ try:
     for x in sys.argv[2:]:
         x = os.path.join("bin", x)
         try:
-            exec(compile(open(x, "rb").read(), x, 'exec'))
+            exec(compile(open(x).read(), x, 'exec'), {'__file__': x})
         except Stop:
             pass
 finally:
@@ -26,12 +26,14 @@ finally:
 dest = sys.argv[1]
 for x in path:
     x = os.path.relpath(x)
+    if not dest:
+        print(x)
+        continue
     d = os.path.join(dest, x)
     try:
         os.mkdir(os.path.basename(x))
-    except OSError as e:
-        if e.errno != errno.EEXIST:
-            raise
+    except FileExistsError:
+        pass
     if os.path.isdir(x):
         shutil.copytree(x, d, True)
     else:

obs/re6st/make

-#!/usr/bin/env python
+#!/usr/bin/env python3
 #
 # Copyright (C) 2016 Julien Muchembled <jm@nexedi.com>
 #
@@ -135,20 +135,20 @@ class task(_task):
         return task_gen
 
     def __init__(self, run, depends, provides=(),
-                 __str_or_task = (basestring, _task)):
+                 __str_or_task = (str, _task)):
         self.run = run
         self.depends = []
         f = []
         for x in (depends,) if isinstance(depends, __str_or_task) else depends:
-            (f if isinstance(x, basestring) else self.depends).append(x)
+            (f if isinstance(x, str) else self.depends).append(x)
         f and self.depends.append(files(*f))
         self.provides = ((provides,)
-            if isinstance(provides, basestring) or callable(provides)
+            if isinstance(provides, str) or callable(provides)
             else provides)
         self.why = self,
 
     def __str__(self):
-        return self.run.__name__
+        return self.__name__
 
     @property
     def input(self):
@@ -162,25 +162,29 @@ class task(_task):
             deps.append((dep, dep(dry_run)))
             x += dep.outputs
         self.outputs = x = []
+        check_times = True
         for p in self.provides:
             if callable(p):
-                x += p(self)
+                o = p(self)
+                if o is None:
+                    check_times = False
+                else:
+                    x += o
             else:
                 x.append(p)
-        if None not in x:
+        if check_times:
             try:
                 _otime = _task._run(self)
-            except OSError as e:
-                if e.errno != errno.ENOENT:
-                    raise
+            except FileNotFoundError:
+                pass
             else:
                 if deps:
                     self.why = [dep for dep, itime in deps if _otime < itime]
         if self.why:
-            print "# Processing %s: %s -> %s" % (self,
+            print("# Processing %s: %s -> %s" % (self,
                 ", ".join(map(str, self.depends)),
                 ", ".join("<%s>" % x.__name__ if callable(x) else x
-                          for x in self.provides or "?"))
+                          for x in self.provides or "?")))
             if not dry_run:
                 try:
                     self.run(self)
@@ -219,14 +223,18 @@ def main():
 
     sys.modules["make"] = sys.modules.pop(__name__)
     f = args.file
-    tasks = {"__file__": f.name}
-    exec(compile(f.read(), f.name, "exec"), tasks)
+    g = {"__file__": f.name}
+    exec(compile(f.read(), f.name, "exec"), g)
+    tasks = {}
+    for k, v in g.items():
+        if isinstance(v, _task):
+            v.__name__ = k
+            tasks[k] = v
     if args.list:
-        print " ".join(sorted(k for k, v in tasks.iteritems()
-                                if isinstance(v, _task)))
+        print(' '.join(sorted(tasks)))
         return
     for t in args.task:
-        if not isinstance(tasks.get(t), _task):
+        if t not in tasks:
             sys.exit("%s is not a valid task." % t)
     for t in args.task:
         tasks[t](args.dry_run)
@@ -249,18 +257,6 @@ class git(tree):
         dry_run or subprocess.check_call(("git", "clone", self.url, root))
         return _INF
 
-def check_output(*args, **kw):
-    # BBB: 'input' arg is a backport from Python 3.4
-    input = kw.pop("input", None)
-    if input is not None:
-        kw["stdin"] = subprocess.PIPE
-    p = subprocess.Popen(stdout=subprocess.PIPE, *args, **kw)
-    out = p.communicate(input)[0]
-    if p.returncode:
-        raise subprocess.CalledProcessError(p.returncode,
-            args[0] if args else kw["args"])
-    return out
-
 @contextmanager
 def cwd(path):
     p = os.getcwd()
@@ -271,18 +267,13 @@ def cwd(path):
         os.chdir(p)
 
 def mkdir(path):
-    try:
-        os.makedirs(path)
-    except OSError as e:
-        if e.errno != errno.EEXIST or not os.path.isdir(path):
-            raise
+    os.makedirs(path, exist_ok=True)
 
 def remove(path):
     try:
         os.remove(path)
-    except OSError as e:
-        if e.errno != errno.ENOENT:
-            raise
+    except FileNotFoundError:
+        pass
 
 def rmtree(path):
     if os.path.exists(path):
@@ -292,11 +283,9 @@ def rmtree(path):
 def make_tar_gz(path, mtime, xform=(lambda x: x), **kw):
     # Make reproducible tarball. Otherwise, it's really annoying that we can't
     # rely on 'osc status' to know whether there are real changes or not.
-    # BBB: Results differ between Python 2.6 and 2.7 because of tarfile.
     __init__ = gzip.GzipFile.__init__
     listdir = os.listdir
     try:
-        if sys.version_info >= (2, 7):
         gzip.GzipFile.__init__ = lambda *args: __init__(mtime=mtime, *args)
         os.listdir = lambda path: sorted(listdir(path))
         t = tarfile.open(path, "w:gz", **kw)
@@ -314,10 +303,6 @@ def make_tar_gz(path, mtime, xform=(lambda x: x), **kw):
         gzip.GzipFile.__init__ = __init__
         os.listdir = listdir
         t.close()
-    if sys.version_info < (2, 7):
-        with open(path, "r+") as t:
-            t.seek(4)
-            gzip.write32u(t, long(mtime))
 
 if __name__ == "__main__":
     sys.exit(main())

obs/re6st/re6st-node.rpmlintrc

+setBadness('binary-or-shlib-defines-rpath', 0)
+setBadness('wrong-script-interpreter', 0)
+setBadness('summary-not-capitalized', 0)
+setBadness('devel-file-in-non-devel-package', 0)

obs/re6st/re6stnet.spec.in

+%global __brp_mangle_shebangs %{nil}
+%global __brp_python_bytecompile %{nil}
+%define units re6stnet.service re6st-registry.service
+
+Summary:    resilient, scalable, IPv6 network application
+Name:       %PACKAGE%
+Version:    %VERSION%
+Release:    1
+License:    GPLv2+
+Group:      Applications/Internet
+AutoReqProv: no
+BuildRequires: gcc-c++, make, python3, iproute
+#!BuildIgnore: rpmlint-Factory
+Source0: %{name}_%{version}.tar.gz
+Source1: %{name}.rpmlintrc
+Requires:   iproute
+BuildRequires: python3-devel
+# dependencies for compilation of python3
+BuildRequires: libffi-devel
+BuildRequires: (lzma-devel or liblzma-devel or xz-devel)
+BuildRequires: zlib-devel
+BuildRequires: (libbz2-devel or bzip2-devel)
+Conflicts:  re6stnet
+
+%description
+%prep
+%setup -q
+
+%build
+make
+
+%install
+set $RPM_BUILD_ROOT
+make install PREFIX=%_prefix MANDIR=%_mandir DESTDIR=$1 %{?_unitdir:UNITDIR=%{_unitdir}}
+# Exclude man pages because they will be compressed.
+find $1 -mindepth 1 -path \*%_mandir -prune -o \
+  -name re6st\* -prune -printf /%%P\\n > INSTALLED
+export QA_RPATHS=$(( 0x0001|0x0002|0x0020 ))
+
+%clean
+rm -rf "$RPM_BUILD_ROOT" INSTALLED
+
+%files -f INSTALLED
+%_mandir/*/*
+
+%post
+if [ $1 -eq 1 ]; then
+    /bin/systemctl preset %{units} || :
+fi >/dev/null 2>&1
+
+%preun
+if [ $1 -eq 0 ]; then
+    /bin/systemctl --no-reload disable %{units} || :
+    /bin/systemctl stop %{units} || :
+fi >/dev/null 2>&1
+find %TARGET% -type d -name __pycache__ -print0 |xargs -r0 rm -r
+
+%postun
+/bin/systemctl daemon-reload >/dev/null 2>&1 || :
+if [ $1 -ge 1 ]; then
+    /bin/systemctl try-restart %{units} >/dev/null 2>&1 || :
+fi