Commit 4a2b7e08 authored by Jérome Perrin's avatar Jérome Perrin

prune: prune dependencies recursively

prune was not recursive when checking dependencies, for example when we
have for example a shared part for openssl that is used only by a shared
part for curl that is not used, we had to run prune twice, the first
iteration would remove curl and only the second iteration would see that
openssl is not used.

This prune in a loop until nothing is pruned by the last iteration. To
support dry-run, we added a ignored_shared_parts parameter to ignore the
parts that are supposed to be deleted.
parent fa0ebacf
Pipeline #9094 passed with stage
in 0 seconds
......@@ -68,29 +68,64 @@ class PruneCommand(ConfigCommand):
sys.exit(do_prune(self.app.log, options, args.dry_run))
def do_prune(logger, options, dry_run):
shared_root = options['shared_part_list'].splitlines()[-1].strip()
logger.warning("Pruning shared directories at %s", shared_root)
def _prune(
logger,
shared_root,
software_root,
instance_root,
ignored_shared_parts,
dry_run,
):
signatures = getUsageSignatureFromSoftwareAndSharedPart(
logger, options['software_root'], shared_root)
logger, software_root, shared_root, ignored_shared_parts)
# recursively look in instance
signatures.update(getUsageSignaturesFromSubInstance(logger, options['instance_root']))
signatures.update(getUsageSignaturesFromSubInstance(logger, instance_root))
for shared_part in glob.glob(os.path.join(shared_root, '*', '*')):
logger.debug("checking shared part %s", shared_part)
h = os.path.basename(shared_part)
for soft, installed_cfg in signatures.items():
if h in installed_cfg:
logger.debug("It is used in %s", soft)
break
else:
if not dry_run:
rmtree(shared_part)
logger.warning(
'Unusued shared parts at %s%s', shared_part,
'' if dry_run else ' ... removed')
if shared_part not in ignored_shared_parts:
logger.debug("checking shared part %s", shared_part)
h = os.path.basename(shared_part)
for soft, installed_cfg in signatures.items():
if h in installed_cfg:
logger.debug("It is used in %s", soft)
break
else:
if not dry_run:
rmtree(shared_part)
logger.warning(
'Unusued shared parts at %s%s', shared_part,
'' if dry_run else ' ... removed')
yield shared_part
def _prune_loop(logger, shared_root, software_root, instance_root, dry_run):
ignored_shared_parts = set([])
while True:
pruned = list(
_prune(
logger,
shared_root,
software_root,
instance_root,
ignored_shared_parts,
dry_run,
))
ignored_shared_parts.update(pruned)
if not pruned:
break
def do_prune(logger, options, dry_run):
shared_root = options['shared_part_list'].splitlines()[-1].strip()
logger.warning("Pruning shared directories at %s", shared_root)
_prune_loop(
logger,
shared_root,
options['software_root'],
options['instance_root'],
dry_run,
)
def getUsageSignaturesFromSubInstance(logger, instance_root):
......@@ -110,8 +145,7 @@ def getUsageSignaturesFromSubInstance(logger, instance_root):
getUsageSignatureFromSoftwareAndSharedPart(
logger, cfg['software_root'], shared_root))
signatures.update(
getUsageSignaturesFromSubInstance(logger, cfg['instance_root'])
)
getUsageSignaturesFromSubInstance(logger, cfg['instance_root']))
return signatures
......@@ -140,10 +174,14 @@ def readSlaposCfg(logger, path):
def getUsageSignatureFromSoftwareAndSharedPart(
logger, software_root, shared_root):
logger, software_root, shared_root, ignored_shared_parts=None):
"""Look in all softwares and shared parts to collect the signatures
that are used.
`ignored_shared_parts` is useful during dry-run, we want to ignore
already the parts that we are about to delete.
"""
if ignored_shared_parts is None:
ignored_shared_parts = set([])
signatures = {}
for installed_cfg in glob.glob(os.path.join(software_root, '*',
'.installed.cfg')):
......@@ -154,13 +192,14 @@ def getUsageSignatureFromSoftwareAndSharedPart(
signatures[script] = f.read()
if shared_root:
for shared_signature in glob.glob(os.path.join(shared_root, '*', '*',
'.*signature')):
with open(shared_signature) as f:
signatures[shared_signature] = f.read()
'.*signature')):
if not any(shared_signature.startswith(ignored_shared_part)
for ignored_shared_part in ignored_shared_parts):
with open(shared_signature) as f:
signatures[shared_signature] = f.read()
return signatures
# XXX copied from https://lab.nexedi.com/nexedi/erp5/blob/31804f683fd36322fb38aeb9654bee70cebe4fdb/erp5/util/testnode/Utils.py
# TODO: move to shared place or ... isn't there already such an utility function in slapos.core ?
import shutil
......
......@@ -110,6 +110,27 @@ class TestPrune(unittest.TestCase):
self.logger.warning.assert_called_with(
'Unusued shared parts at %s%s', not_used, ' ... removed')
def test_shared_part_not_used_recursive_dependencies(self):
used_only_by_orphean_part = self._createSharedPart(
'used_only_by_orphean_part')
not_used = self._createSharedPart(
'not_used', using=used_only_by_orphean_part)
used_directly = self._createSharedPart('used_directly')
self._createFakeSoftware(self.id(), using=used_directly)
do_prune(self.logger, self.config, True)
self.assertIn(
mock.call('Unusued shared parts at %s%s', not_used, ''),
self.logger.warning.mock_calls)
self.assertIn(
mock.call(
'Unusued shared parts at %s%s', used_only_by_orphean_part, ''),
self.logger.warning.mock_calls)
do_prune(self.logger, self.config, False)
self.assertTrue(os.path.exists(used_directly))
self.assertFalse(os.path.exists(not_used))
self.assertFalse(os.path.exists(used_only_by_orphean_part))
def test_shared_part_used_in_buildout_script(self):
not_used = self._createSharedPart('not_used')
used_in_script = self._createSharedPart('used_in_script')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment