Commit aa526c95 authored by Jérome Perrin's avatar Jérome Perrin

check_software: check #! in scripts installed by softwares

Some software install scripts with a #!/ line refering to a system
program, such as for example #!/usr/bin/python . Later when they are
executed this may fail because the system program might not be present
or might not be the wrong version.

This extends the software check to detect these and flag them as errors.
Software release authors will have to adjust the generated scripts, for
example by using post build step.
parent 50215b31
Pipeline #21216 failed with stage
in 0 seconds
...@@ -28,6 +28,7 @@ ...@@ -28,6 +28,7 @@
import fnmatch import fnmatch
import glob import glob
import io
import os import os
import re import re
import warnings import warnings
...@@ -86,6 +87,12 @@ def checkSoftware(slap, software_url): ...@@ -86,6 +87,12 @@ def checkSoftware(slap, software_url):
'liblibgolang', 'liblibgolang',
)) ))
# Programs allowed in #! "shebang" lines for scripts executables
system_programs_allowed_list = set((
'/bin/bash',
'/bin/sh',
))
# we also ignore a few patterns for part that are known to be binary distributions, # we also ignore a few patterns for part that are known to be binary distributions,
# for which we generate LD_LIBRARY_PATH wrappers or we don't use directly. # for which we generate LD_LIBRARY_PATH wrappers or we don't use directly.
ignored_file_patterns = set(( ignored_file_patterns = set((
...@@ -106,14 +113,36 @@ def checkSoftware(slap, software_url): ...@@ -106,14 +113,36 @@ def checkSoftware(slap, software_url):
'*/node_modules/phantomjs*/*', '*/node_modules/phantomjs*/*',
'*/grafana/tools/phantomjs/*', '*/grafana/tools/phantomjs/*',
'*/node_modules/puppeteer/*', '*/node_modules/puppeteer/*',
# scripts in node packages all have #!/usr/bin/env node, but we never use
# them directly and generate wrappers.
'*/node_modules/bin/*',
# left over of compilation failures # left over of compilation failures
'*/*__compile__/*', '*/*__compil__/*',
# build dir for packages built in-place # build dir for packages built in-place
'*/parts/wendelin.core/build/*', '*/parts/wendelin.core/build/*',
# git checkouts containing scripts that we don't use
'*/parts/erp5/product/ERP5/bin/*',
# plain downloads containing scripts that we don't use
'*/parts/wxPython-download/bin/make-new-etg-file.py',
'*/parts/wxPython-download/bin/make-new-unittest-file.py',
'*/ant/*/bin/antRun.pl',
'*/ant/*/bin/complete-ant-cmd.pl',
'*/ant/*/bin/runant.pl',
'*/ant/*/bin/runant.py',
'*/nextcloud-app-news/*/vendor/bin/feedio',
'*/nextcloud-app-news/*/vendor/debril/feed-io/bin/feedio',
# the depot_tools package used to build Chromium installs some # the depot_tools package used to build Chromium installs some
# Python libraries lacking an rpath; these are not actually used # Python libraries lacking an rpath; these are not actually used
# by Chromium itself # by Chromium itself
'*/.vpython-root/*', '*/.vpython-root/*',
# depots tools also contain several scripts with wrong shebang that are not used
# neither during build or runtime
'*/parts/depot_tools/bootstrap-2@3.8.10.chromium.23_bin/*/bin/*',
)) ))
software_hash = md5digest(software_url) software_hash = md5digest(software_url)
...@@ -176,26 +205,34 @@ def checkSoftware(slap, software_url): ...@@ -176,26 +205,34 @@ def checkSoftware(slap, software_url):
**locals())) **locals()))
return libraries return libraries
def checkExecutableLink(paths_to_check, valid_paths_for_libs): def checkExecutables(paths_to_check, valid_paths):
# type: (Iterable[str], Iterable[str]) -> List[str] # type: (Iterable[str], Iterable[str]) -> List[str]
"""Check shared libraries linked with executables in `paths_to_check`. """Check executables in `paths_to_check`.
Only libraries from `valid_paths_for_libs` are accepted.
When executables are ELF binaries, we check that there are only linked
with libraries from `valid_paths`.
When executables are scripts, we look at their #!/ "shebang" lines and
check they are using executables from `valid_path_for_programs`.
Returns a list of error messages. Returns a list of error messages.
""" """
valid_paths_for_libs = [os.path.realpath(x) for x in valid_paths_for_libs] valid_paths = [os.path.realpath(x) for x in valid_paths]
executable_link_error_list = [] executable_error_list = []
for path in paths_to_check: for path in paths_to_check:
for root, dirs, files in os.walk(path): for root, dirs, files in os.walk(path):
is_bin_dir = root.split(os.sep)[-1] in ('bin', 'sbin')
for f in files: for f in files:
f = os.path.join(root, f) f = os.path.join(root, f)
if any(fnmatch.fnmatch(f, ignored_pattern) if any(fnmatch.fnmatch(f, ignored_pattern)
for ignored_pattern in ignored_file_patterns): for ignored_pattern in ignored_file_patterns):
continue continue
if os.access(f, os.X_OK) or fnmatch.fnmatch(f, '*.so'): executable = os.access(f, os.X_OK)
if executable or fnmatch.fnmatch(f, '*.so'):
try: try:
libs = getLddOutput(f) libs = getLddOutput(f)
except DynamicLibraryNotFound as e: except DynamicLibraryNotFound as e:
executable_link_error_list.append(str(e)) executable_error_list.append(str(e))
else: else:
for lib, lib_path in libs.items(): for lib, lib_path in libs.items():
if lib.split('.')[0] in system_lib_allowed_list: if lib.split('.')[0] in system_lib_allowed_list:
...@@ -204,16 +241,31 @@ def checkSoftware(slap, software_url): ...@@ -204,16 +241,31 @@ def checkSoftware(slap, software_url):
# dynamically linked programs can only be linked with libraries # dynamically linked programs can only be linked with libraries
# present in software or in shared parts repository. # present in software or in shared parts repository.
if any(lib_path.startswith(valid_path) if any(lib_path.startswith(valid_path)
for valid_path in valid_paths_for_libs): for valid_path in valid_paths):
continue continue
executable_link_error_list.append( executable_error_list.append(
'{f} uses system library {lib_path} for {lib}'.format( '{f} uses system library {lib_path} for {lib}'.format(
**locals())) **locals()))
return executable_link_error_list if executable and is_bin_dir:
with (io.open if six.PY2 else open)(f, 'r', encoding='utf-8', errors='ignore') as ft:
shebang_line = ft.readline().strip()
if shebang_line[:3] == '#!/':
program = shebang_line[2:].split()[0].strip()
if not (
any(program.startswith(p) for p in valid_paths)
or program in system_programs_allowed_list):
executable_error_list.append(
'{f} uses {shebang_line}'.format(**locals()))
return executable_error_list
software_directory = os.path.join(slap.software_directory, software_hash) software_directory = os.path.join(slap.software_directory, software_hash)
paths_to_check = set((software_directory, )) paths_to_check = set((software_directory, ))
# bin/buildout is an exception for the #! check, it might be using
# slapos.core python.
ignored_file_patterns.add(os.path.join(software_directory, 'bin', 'buildout'))
# Compute the paths to check by inspecting buildout installed database # Compute the paths to check by inspecting buildout installed database
# for this software. We are looking for shared parts installed by recipes. # for this software. We are looking for shared parts installed by recipes.
config_parser = ConfigParser() config_parser = ConfigParser()
...@@ -226,7 +278,7 @@ def checkSoftware(slap, software_url): ...@@ -226,7 +278,7 @@ def checkSoftware(slap, software_url):
paths_to_check.add(section_path) paths_to_check.add(section_path)
error_list.extend( error_list.extend(
checkExecutableLink( checkExecutables(
paths_to_check, paths_to_check,
tuple(paths_to_check) + tuple(slap._shared_part_list), tuple(paths_to_check) + tuple(slap._shared_part_list),
)) ))
......
...@@ -213,3 +213,64 @@ class TestCheckSoftwareEggVulnerability(SlapOSStandaloneTestCase): ...@@ -213,3 +213,64 @@ class TestCheckSoftwareEggVulnerability(SlapOSStandaloneTestCase):
'Lxml 4.6.5 includes a fix for CVE-2021-43818', 'Lxml 4.6.5 includes a fix for CVE-2021-43818',
str(warning.message), str(warning.message),
) )
class TestCheckSoftwareShebang(SlapOSStandaloneTestCase):
def _make_script(self, path, shebang):
"""Install a dummy executable script at `path` using `shebang` line.
"""
with open(path, 'w') as f:
f.write("#!%s\n" % shebang)
os.chmod(path, 0o700)
def _install_software(self):
"""install a fake software with scripts.
"""
software_url = '/fake/path/software.cfg'
software_hash = md5digest(software_url)
self.standalone.supply(software_url)
software_dir = os.path.join(
self.standalone.software_directory,
software_hash,
)
program_bin_dir = os.path.join(software_dir, 'parts', 'program', 'bin')
os.makedirs(program_bin_dir)
program_sbin_dir = os.path.join(software_dir, 'parts', 'program', 'sbin')
os.makedirs(program_sbin_dir)
program_ignored_dir = os.path.join(
software_dir, 'parts', 'program', 'ignored')
os.makedirs(program_ignored_dir)
# ok scripts
self._make_script(os.path.join(program_bin_dir, 'ok-using-sh'), '/bin/sh')
self._make_script(
os.path.join(program_bin_dir, 'ok-using-another-program'),
program_bin_dir + '/ok')
self._make_script(
os.path.join(program_ignored_dir, 'ok-not-in-bin'), '/not/allowed')
# error scripts
self._make_script(
os.path.join(program_bin_dir, 'error-not-allowed'), '/not/allowed')
self._make_script(
os.path.join(program_sbin_dir, 'error-not-allowed-sbin'), '/not/allowed')
with open(os.path.join(software_dir, '.completed'), 'w') as f:
f.write('Thu Dec 2 01:35:02 2021')
with open(os.path.join(software_dir, '.installed.cfg'), 'w') as f:
f.write('[buildout]')
return software_url
def test_shebang(self):
software_url = self._install_software()
with self.assertRaises(RuntimeError) as e:
checkSoftware(self.standalone, software_url)
errors = sorted(e.exception.args[0].splitlines())
self.assertIn(
'parts/program/bin/error-not-allowed uses #!/not/allowed', errors[0])
self.assertIn(
'parts/program/sbin/error-not-allowed-sbin uses #!/not/allowed',
errors[1])
self.assertEqual(len(errors), 2, errors)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment