check_software fixes

 - only consider shared parts from checked software
 - support new `.buildout-shared.json` signature files
 - unit test coverage

See merge request nexedi/slapos.core!361

slapos/testing/check_software.py

@@ -34,6 +34,7 @@ import warnings
 
 import pkg_resources
 import requests
+from six.moves.configparser import ConfigParser
 
 try:
   import subprocess32 as subprocess
@@ -209,19 +210,34 @@ def checkSoftware(slap, software_url):
                         **locals()))
     return executable_link_error_list
 
-  paths_to_check = (
-      os.path.join(slap.software_directory, software_hash),
-      slap.shared_directory,
-  )
+  software_directory = os.path.join(slap.software_directory, software_hash)
+  paths_to_check = set((software_directory, ))
+
+  # Compute the paths to check by inspecting buildout installed database
+  # for this software. We are looking for shared parts installed by recipes.
+  config_parser = ConfigParser()
+  config_parser.read(os.path.join(software_directory, '.installed.cfg'))
+  for section_name in config_parser.sections():
+    for option_name in 'location', '__buildout_installed__':
+      if config_parser.has_option(section_name, option_name):
+         for section_path in config_parser.get(section_name, option_name).splitlines():
+           if section_path and not section_path.startswith(software_directory):
+             paths_to_check.add(section_path)
+
   error_list.extend(
       checkExecutableLink(
           paths_to_check,
-          paths_to_check + tuple(slap._shared_part_list),
+          tuple(paths_to_check) + tuple(slap._shared_part_list),
       ))
 
   # check this software is not referenced in any shared parts.
-  for signature_file in glob.glob(os.path.join(slap.shared_directory, '*', '*',
-                                               '.*slapos.*.signature')):
+  for signature_file in glob.glob(
+      os.path.join(
+          slap.shared_directory,
+          '*',
+          '*',
+          '.buildout-shared.json',
+      )):
     with open(signature_file) as f:
       signature_content = f.read()
     if software_hash in signature_content:

slapos/tests/data/cmmi/Makefile

+LDLIBS = -lz
+include Makefile.conf
+
+.PHONY: all install uninstall clean
+
+all: check
+
+slapos-core-test: main.c
+	$(CC) $(CPPFLAGS) $(CFLAGS) $(LDFLAGS) -o $@ $< $(LDLIBS)
+
+install: all
+	install -Dp slapos-core-test $(DESTDIR)$(PREFIX)/bin/slapos-core-test
+
+clean:
+	-rm -f slapos-core-test
+
+check: slapos-core-test
+	./slapos-core-test
+
+dist: main.c Makefile configure
+	tar --transform 's,^,slapos-core-test-0.0.0/,' -czvf dist.tar.gz $?

slapos/tests/data/cmmi/configure

+#!/bin/sh
+
+echo "configured with: $*"
+echo "PREFIX=$(echo "$1" | sed 's/^[^=]*=//g')" > Makefile.conf

slapos/tests/data/cmmi/main.c

+#include "zlib.h"
+#include <stdio.h>
+
+int main(int argc, char *argv[]){
+    printf("%s: using zlib: %s\n", argv[0], zlibVersion());
+    return 0;
+}
+

slapos/tests/test_check_software.py

+##############################################################################
+#
+# Copyright (c) 2021 Vifib SARL and Contributors. All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License
+# as published by the Free Software Foundation; either version 3
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+from __future__ import absolute_import
+import unittest
+import mock
+import os
+import glob
+import tempfile
+import textwrap
+
+from slapos.testing.check_software import checkSoftware
+
+from .test_standalone import SlapOSStandaloneTestCase
+
+
+class TestCheckSoftware(SlapOSStandaloneTestCase):
+  # BBB python2
+  assertRaisesRegex = getattr(
+      unittest.TestCase,
+      'assertRaisesRegex',
+      unittest.TestCase.assertRaisesRegexp,
+  )
+
+  def _get_zlib_environment(self, with_rpath=True):
+    """returns an environment that will compile with slapos' zlib
+    """
+    # find a zlib in SLAPOS_TEST_SHARED_PART_LIST
+    zlib_location = None
+    for shared_part in self.standalone._shared_part_list:
+      for zlib_location in glob.glob(os.path.join(shared_part, 'zlib', '*')):
+        break
+      if zlib_location:
+        break
+    assert zlib_location
+    return {
+        'CFLAGS':
+            '-I{zlib_location}/include'.format(**locals()),
+        'LDFLAGS':
+            '-L{zlib_location}/lib -Wl,-rpath={zlib_location}/lib'.format(
+                **locals()) if with_rpath else '-L{zlib_location}/lib'.format(
+                    **locals())
+    }
+
+  def _install_software(self, environment, shared=True):
+    environment_option = ''
+    for k, v in environment.items():
+      environment_option += '  {k}={v}\n'.format(k=k, v=v)
+
+    shared_option = 'true' if shared else 'false'
+    test_software_archive_url = os.path.join(
+        os.path.dirname(__file__), 'data', 'cmmi', 'dist.tar.gz')
+    with tempfile.NamedTemporaryFile(
+        suffix="-%s.cfg" % self.id(),
+        mode='w',
+    ) as f:
+      f.write(
+          textwrap.dedent('''
+              [buildout]
+              parts = cmmi
+              newest = false
+              offline = true
+              eggs-directory = {os.environ[SLAPOS_TEST_EGGS_DIRECTORY]}
+              develop-eggs-directory = {os.environ[SLAPOS_TEST_DEVELOP_EGGS_DIRECTORY]}
+ 
+              [cmmi]
+              recipe = slapos.recipe.cmmi
+              url = {test_software_archive_url}
+              shared = {shared_option}
+              environment =
+              {environment_option}
+      ''').format(os=os, **locals()))
+      f.flush()
+      self.standalone.supply(f.name)
+      self.standalone.waitForSoftware()
+      return f.name
+
+  def test_software_using_system_libraries(self):
+    software_url = self._install_software(
+        self._get_zlib_environment(with_rpath=False), shared=False)
+    with self.assertRaisesRegex(
+        RuntimeError,
+        './bin/slapos-core-test uses system library .*libz.so.* for libz.so',
+    ):
+      checkSoftware(self.standalone, software_url)
+
+  def test_shared_part_using_system_libraries(self):
+    software_url = self._install_software(
+        self._get_zlib_environment(with_rpath=False))
+    with self.assertRaisesRegex(
+        RuntimeError,
+        './bin/slapos-core-test uses system library .*libz.so.* for libz.so',
+    ):
+      checkSoftware(self.standalone, software_url)
+
+  def test_shared_part_referencing_software(self):
+    environment = self._get_zlib_environment()
+    environment['reference-to-part'] = '${buildout:parts-directory}'
+    software_url = self._install_software(environment=environment)
+    with self.assertRaisesRegex(
+        RuntimeError,
+        'Software hash present in signature',
+    ):
+      checkSoftware(self.standalone, software_url)
+
+  def test_ok(self):
+    software_url = self._install_software(
+        environment=self._get_zlib_environment())
+    checkSoftware(self.standalone, software_url)
+
+  def test_software_check_isolated(self):
+    # if a software populated shared parts with wrong parts, this does not
+    # impact checking other softwares, as long as they don't use the problematic
+    # parts
+    self.test_shared_part_using_system_libraries()
+    self.test_ok()

slapos/tests/test_standalone.py

@@ -30,7 +30,6 @@ import mock
 import os
 import tempfile
 import textwrap
-import shutil
 import hashlib
 import socket
 import errno
@@ -47,6 +46,8 @@ from slapos.slap.standalone import StandaloneSlapOS
 from slapos.slap.standalone import SlapOSNodeSoftwareError
 from slapos.slap.standalone import PartitionForwardConfiguration
 from slapos.slap.standalone import PartitionForwardAsPartitionConfiguration
+import slapos.util
+
 
 SLAPOS_TEST_IPV4 = os.environ['SLAPOS_TEST_IPV4']
 SLAPOS_TEST_IPV6 = os.environ['SLAPOS_TEST_IPV6']
@@ -80,7 +81,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
 
   def test_format(self):
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     standalone = StandaloneSlapOS(
         working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
     self.addCleanup(standalone.stop)
@@ -100,7 +101,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
 
   def test_reformat_less_partitions(self):
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     standalone = StandaloneSlapOS(
         working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
     self.addCleanup(standalone.stop)
@@ -115,7 +116,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
 
   def test_reformat_less_chmod_files(self):
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     standalone = StandaloneSlapOS(
         working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
     self.addCleanup(standalone.stop)
@@ -132,7 +133,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
 
   def test_reformat_different_base_name(self):
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     standalone = StandaloneSlapOS(
         working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
     self.addCleanup(standalone.stop)
@@ -152,7 +153,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
 
   def test_reformat_refuse_deleting_running_partition(self):
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     standalone = StandaloneSlapOS(
         working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
     self.addCleanup(standalone.stop)
@@ -163,7 +164,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
 
   def test_slapos_node_format(self):
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     standalone = StandaloneSlapOS(
         working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
     self.addCleanup(standalone.stop)
@@ -180,13 +181,13 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
       partitions = glob.glob(glob_pattern)
       self.assertEqual(partition_count, len(partitions))
       for path in partitions:
-        shutil.rmtree(path)
+        slapos.util.rmtree(path)
       subprocess.check_call(format_command)
       self.assertEqual(partition_count, len(glob.glob(glob_pattern)))
 
   def test_two_instance_from_same_directory(self):
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     standalone1 = StandaloneSlapOS(
         working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
 
@@ -213,7 +214,7 @@ class TestSlapOSStandaloneSetup(unittest.TestCase):
   def test_partition_forward(self):
     # type: () -> None
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     partition_forward_config = [
         PartitionForwardConfiguration(
             'https://slapos1.example.com',
@@ -303,9 +304,16 @@ class SlapOSStandaloneTestCase(unittest.TestCase):
   def setUp(self):
     checkPortIsFree()
     working_dir = tempfile.mkdtemp(prefix=__name__)
-    self.addCleanup(shutil.rmtree, working_dir)
+    self.addCleanup(slapos.util.rmtree, working_dir)
     self.standalone = StandaloneSlapOS(
-        working_dir, SLAPOS_TEST_IPV4, SLAPOS_TEST_PORT)
+        working_dir,
+        SLAPOS_TEST_IPV4,
+        SLAPOS_TEST_PORT,
+        shared_part_list=[
+            os.path.expanduser(p) for p in os.environ.get(
+                'SLAPOS_TEST_SHARED_PART_LIST', '').split(os.pathsep) if p
+        ],
+    )
     if self._auto_stop_standalone:
       self.addCleanup(self.standalone.stop)
     self.standalone.format(1, SLAPOS_TEST_IPV4, SLAPOS_TEST_IPV6)