Commit 4a2b7e08 authored by Jérome Perrin's avatar Jérome Perrin

prune: prune dependencies recursively

prune was not recursive when checking dependencies, for example when we
have for example a shared part for openssl that is used only by a shared
part for curl that is not used, we had to run prune twice, the first
iteration would remove curl and only the second iteration would see that
openssl is not used.

This prune in a loop until nothing is pruned by the last iteration. To
support dry-run, we added a ignored_shared_parts parameter to ignore the
parts that are supposed to be deleted.
parent fa0ebacf
Pipeline #9094 passed with stage
in 0 seconds
...@@ -68,29 +68,64 @@ class PruneCommand(ConfigCommand): ...@@ -68,29 +68,64 @@ class PruneCommand(ConfigCommand):
sys.exit(do_prune(self.app.log, options, args.dry_run)) sys.exit(do_prune(self.app.log, options, args.dry_run))
def do_prune(logger, options, dry_run): def _prune(
shared_root = options['shared_part_list'].splitlines()[-1].strip() logger,
logger.warning("Pruning shared directories at %s", shared_root) shared_root,
software_root,
instance_root,
ignored_shared_parts,
dry_run,
):
signatures = getUsageSignatureFromSoftwareAndSharedPart( signatures = getUsageSignatureFromSoftwareAndSharedPart(
logger, options['software_root'], shared_root) logger, software_root, shared_root, ignored_shared_parts)
# recursively look in instance # recursively look in instance
signatures.update(getUsageSignaturesFromSubInstance(logger, options['instance_root'])) signatures.update(getUsageSignaturesFromSubInstance(logger, instance_root))
for shared_part in glob.glob(os.path.join(shared_root, '*', '*')): for shared_part in glob.glob(os.path.join(shared_root, '*', '*')):
logger.debug("checking shared part %s", shared_part) if shared_part not in ignored_shared_parts:
h = os.path.basename(shared_part) logger.debug("checking shared part %s", shared_part)
for soft, installed_cfg in signatures.items(): h = os.path.basename(shared_part)
if h in installed_cfg: for soft, installed_cfg in signatures.items():
logger.debug("It is used in %s", soft) if h in installed_cfg:
break logger.debug("It is used in %s", soft)
else: break
if not dry_run: else:
rmtree(shared_part) if not dry_run:
logger.warning( rmtree(shared_part)
'Unusued shared parts at %s%s', shared_part, logger.warning(
'' if dry_run else ' ... removed') 'Unusued shared parts at %s%s', shared_part,
'' if dry_run else ' ... removed')
yield shared_part
def _prune_loop(logger, shared_root, software_root, instance_root, dry_run):
ignored_shared_parts = set([])
while True:
pruned = list(
_prune(
logger,
shared_root,
software_root,
instance_root,
ignored_shared_parts,
dry_run,
))
ignored_shared_parts.update(pruned)
if not pruned:
break
def do_prune(logger, options, dry_run):
shared_root = options['shared_part_list'].splitlines()[-1].strip()
logger.warning("Pruning shared directories at %s", shared_root)
_prune_loop(
logger,
shared_root,
options['software_root'],
options['instance_root'],
dry_run,
)
def getUsageSignaturesFromSubInstance(logger, instance_root): def getUsageSignaturesFromSubInstance(logger, instance_root):
...@@ -110,8 +145,7 @@ def getUsageSignaturesFromSubInstance(logger, instance_root): ...@@ -110,8 +145,7 @@ def getUsageSignaturesFromSubInstance(logger, instance_root):
getUsageSignatureFromSoftwareAndSharedPart( getUsageSignatureFromSoftwareAndSharedPart(
logger, cfg['software_root'], shared_root)) logger, cfg['software_root'], shared_root))
signatures.update( signatures.update(
getUsageSignaturesFromSubInstance(logger, cfg['instance_root']) getUsageSignaturesFromSubInstance(logger, cfg['instance_root']))
)
return signatures return signatures
...@@ -140,10 +174,14 @@ def readSlaposCfg(logger, path): ...@@ -140,10 +174,14 @@ def readSlaposCfg(logger, path):
def getUsageSignatureFromSoftwareAndSharedPart( def getUsageSignatureFromSoftwareAndSharedPart(
logger, software_root, shared_root): logger, software_root, shared_root, ignored_shared_parts=None):
"""Look in all softwares and shared parts to collect the signatures """Look in all softwares and shared parts to collect the signatures
that are used. that are used.
`ignored_shared_parts` is useful during dry-run, we want to ignore
already the parts that we are about to delete.
""" """
if ignored_shared_parts is None:
ignored_shared_parts = set([])
signatures = {} signatures = {}
for installed_cfg in glob.glob(os.path.join(software_root, '*', for installed_cfg in glob.glob(os.path.join(software_root, '*',
'.installed.cfg')): '.installed.cfg')):
...@@ -154,13 +192,14 @@ def getUsageSignatureFromSoftwareAndSharedPart( ...@@ -154,13 +192,14 @@ def getUsageSignatureFromSoftwareAndSharedPart(
signatures[script] = f.read() signatures[script] = f.read()
if shared_root: if shared_root:
for shared_signature in glob.glob(os.path.join(shared_root, '*', '*', for shared_signature in glob.glob(os.path.join(shared_root, '*', '*',
'.*signature')): '.*signature')):
with open(shared_signature) as f: if not any(shared_signature.startswith(ignored_shared_part)
signatures[shared_signature] = f.read() for ignored_shared_part in ignored_shared_parts):
with open(shared_signature) as f:
signatures[shared_signature] = f.read()
return signatures return signatures
# XXX copied from https://lab.nexedi.com/nexedi/erp5/blob/31804f683fd36322fb38aeb9654bee70cebe4fdb/erp5/util/testnode/Utils.py # XXX copied from https://lab.nexedi.com/nexedi/erp5/blob/31804f683fd36322fb38aeb9654bee70cebe4fdb/erp5/util/testnode/Utils.py
# TODO: move to shared place or ... isn't there already such an utility function in slapos.core ? # TODO: move to shared place or ... isn't there already such an utility function in slapos.core ?
import shutil import shutil
......
...@@ -110,6 +110,27 @@ class TestPrune(unittest.TestCase): ...@@ -110,6 +110,27 @@ class TestPrune(unittest.TestCase):
self.logger.warning.assert_called_with( self.logger.warning.assert_called_with(
'Unusued shared parts at %s%s', not_used, ' ... removed') 'Unusued shared parts at %s%s', not_used, ' ... removed')
def test_shared_part_not_used_recursive_dependencies(self):
used_only_by_orphean_part = self._createSharedPart(
'used_only_by_orphean_part')
not_used = self._createSharedPart(
'not_used', using=used_only_by_orphean_part)
used_directly = self._createSharedPart('used_directly')
self._createFakeSoftware(self.id(), using=used_directly)
do_prune(self.logger, self.config, True)
self.assertIn(
mock.call('Unusued shared parts at %s%s', not_used, ''),
self.logger.warning.mock_calls)
self.assertIn(
mock.call(
'Unusued shared parts at %s%s', used_only_by_orphean_part, ''),
self.logger.warning.mock_calls)
do_prune(self.logger, self.config, False)
self.assertTrue(os.path.exists(used_directly))
self.assertFalse(os.path.exists(not_used))
self.assertFalse(os.path.exists(used_only_by_orphean_part))
def test_shared_part_used_in_buildout_script(self): def test_shared_part_used_in_buildout_script(self):
not_used = self._createSharedPart('not_used') not_used = self._createSharedPart('not_used')
used_in_script = self._createSharedPart('used_in_script') used_in_script = self._createSharedPart('used_in_script')
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment