diff --git a/slapos/grid/SlapObject.py b/slapos/grid/SlapObject.py index c48d65fdb2d069041607823d6b30a987e90557d1..81e83f58ee52ed1425a0abc76e97136ebca6f0f9 100644 --- a/slapos/grid/SlapObject.py +++ b/slapos/grid/SlapObject.py @@ -453,6 +453,17 @@ class Partition(object): 'USER': pwd.getpwuid(uid).pw_name, } + def addServiceToCustomGroup(self, group_id, runner_list, path): + """Add new services to supervisord that belong to specific group""" + group_partition_template = pkg_resources.resource_stream(__name__, + 'templates/group_partition_supervisord.conf.in').read() + self.supervisor_configuration_groups += group_partition_template % { + 'instance_id': group_id, + 'program_list': ','.join(['_'.join([group_id, runner]) + for runner in runner_list]) + } + return self.addServiceToGroup(group_id, runner_list, path) + def updateSymlink(self, sr_symlink, software_path): if os.path.lexists(sr_symlink): if not os.path.islink(sr_symlink): @@ -589,7 +600,7 @@ class Partition(object): self.generateSupervisorConfigurationFile() self.createRetentionLockDelay() - def generateSupervisorConfigurationFile(self): + def generateSupervisorConfiguration(self): """ Generates supervisord configuration file from template. @@ -600,6 +611,8 @@ class Partition(object): """ runner_list = [] service_list = [] + self.partition_supervisor_configuration = "" + self.supervisor_configuration_groups = "" if os.path.exists(self.run_path): if os.path.isdir(self.run_path): runner_list = os.listdir(self.run_path) @@ -615,7 +628,7 @@ class Partition(object): partition_id = self.computer_partition.getId() group_partition_template = pkg_resources.resource_stream(__name__, 'templates/group_partition_supervisord.conf.in').read() - self.partition_supervisor_configuration = group_partition_template % { + self.supervisor_configuration_groups = group_partition_template % { 'instance_id': partition_id, 'program_list': ','.join(['_'.join([partition_id, runner]) for runner in runner_list + service_list]) @@ -624,10 +637,25 @@ class Partition(object): self.addServiceToGroup(partition_id, runner_list, self.run_path) self.addServiceToGroup(partition_id, service_list, self.service_path, extension=WATCHDOG_MARK) + + def writeSupervisorConfigurationFile(self): + """ + Write supervisord configuration file and update supervisord + """ + if self.supervisor_configuration_groups and \ + self.partition_supervisor_configuration: updateFile(self.supervisord_partition_configuration_path, + self.supervisor_configuration_groups + self.partition_supervisor_configuration) self.updateSupervisor() + def generateSupervisorConfigurationFile(self): + """ + update supervisord with new processes + """ + self.generateSupervisorConfiguration() + self.writeSupervisorConfigurationFile() + def start(self): """Asks supervisord to start the instance. If this instance is not installed, we install it. @@ -720,6 +748,19 @@ class Partition(object): return True + def checkProcessesFromStateList(self, process_list, state_list): + """Asks supervisord to check if one of the processes are in the state_list.""" + supervisor = self.getSupervisorRPC() + for process in process_list: + try: + info = supervisor.getProcessInfo(process) + if info['statename'] in state_list: + return True + except xmlrpclib.Fault as exc: + self.logger.debug("BAD process name: %r" % process) + continue + return False + def cleanupFolder(self, folder_path): """Delete all files and folders in a specified directory """ diff --git a/slapos/grid/slapgrid.py b/slapos/grid/slapgrid.py index fc8d3d86cc599a5eb179b02251a3ad6c4735a107..eac8dd7e42ee497e8a0af200a2436e4cbdc7fa14 100644 --- a/slapos/grid/slapgrid.py +++ b/slapos/grid/slapgrid.py @@ -80,6 +80,7 @@ PROMISE_TIMEOUT = 3 COMPUTER_PARTITION_TIMESTAMP_FILENAME = '.timestamp' COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME = '.slapos_latest_bang_timestamp' COMPUTER_PARTITION_INSTALL_ERROR_FILENAME = '.slapgrid-%s-error.log' +COMPUTER_PARTITION_WAIT_LIST_FILENAME = '.slapos-wait-services' # XXX hardcoded watchdog_path WATCHDOG_PATH = '/opt/slapos/bin/slapos-watchdog' @@ -1266,6 +1267,18 @@ stderr_logfile_backups=1 return SLAPGRID_PROMISE_FAIL return SLAPGRID_SUCCESS + def _checkWaitProcessList(self, partition, state_list): + wait_file = os.path.join(partition.instance_path, + COMPUTER_PARTITION_WAIT_LIST_FILENAME) + + if os.path.exists(wait_file) and os.path.isfile(wait_file): + with open(wait_file) as wait_f: + processes_list = [name.strip() for name in wait_f.readlines() if name] + # return True if one of process in the list is running + return partition.checkProcessesFromStateList(processes_list, + state_list) + return False + def validateXML(self, to_be_validated, xsd_model): """Validates a given xml file""" #We retrieve the xsd model @@ -1572,10 +1585,19 @@ stderr_logfile_backups=1 raise except Exception: pass + # let managers update current partition + for manager in self._manager_list: + manager.report(local_partition) + if computer_partition.getId() in report_usage_issue_cp_list: self.logger.info('Ignoring destruction of %r, as no report usage was sent' % computer_partition.getId()) continue + if self._checkWaitProcessList(local_partition, + state_list=['RUNNING', 'STARTING']): + self.logger.info('There are running processes into the partition,' \ + ' wait until they finish...') + continue destroyed = local_partition.destroy() except (SystemExit, KeyboardInterrupt): computer_partition.error(traceback.format_exc(), logger=self.logger) diff --git a/slapos/manager/interface.py b/slapos/manager/interface.py index 96e54695a5b5b1a21ceb38b607ba1263e4326322..933e08a6c1290d5a0ec930e4dc7c962e2d5c3063 100644 --- a/slapos/manager/interface.py +++ b/slapos/manager/interface.py @@ -29,3 +29,8 @@ class IManager(Interface): :param partition: slapos.grid.SlapObject.Partition, currently processed partition """ + def report(partition): + """Method called at `slapos node report` phase. + + :param partition: slapos.grid.SlapObject.Partition, currently processed partition + """ diff --git a/slapos/manager/predestroy.py b/slapos/manager/predestroy.py new file mode 100644 index 0000000000000000000000000000000000000000..606b3e6bba9427f3bac1fd58906e9451036bfb5a --- /dev/null +++ b/slapos/manager/predestroy.py @@ -0,0 +1,78 @@ +# coding: utf-8 +import logging +import os +import sys +import subprocess + +from zope import interface as zope_interface +from slapos.manager import interface +from slapos.grid.slapgrid import COMPUTER_PARTITION_WAIT_LIST_FILENAME + +logger = logging.getLogger(__name__) +WIPE_WRAPPER_BASE_PATH = "var/run/slapos/pre-destroy/" + +class Manager(object): + """Manager is called in every step of preparation of the computer.""" + + zope_interface.implements(interface.IManager) + + def __init__(self, config): + """Manager needs to know config for its functioning. + """ + pass + + def format(self, computer): + """Method called at `slapos node format` phase. + """ + pass + + def software(self, software): + """Method called at `slapos node software` phase. + """ + pass + + def instance(self, partition): + """Method called at `slapos node instance` phase. + """ + pass + + def report(self, partition): + """Method called at `slapos node report` phase.""" + + partition.createRetentionLockDate() + if not partition.checkRetentionIsAuthorized(): + return + + wait_filepath = os.path.join(partition.instance_path, + COMPUTER_PARTITION_WAIT_LIST_FILENAME) + wipe_base_folder = os.path.join(partition.instance_path, + WIPE_WRAPPER_BASE_PATH) + if not os.path.exists(wipe_base_folder): + return + wipe_wrapper_list = [f for f in os.listdir(wipe_base_folder) + if os.path.isfile(os.path.join(wipe_base_folder, f))] + if len(wipe_wrapper_list) > 0: + group_name = partition.partition_id + '-' + "destroy" + logger.info("Adding pre-destroy scripts to supervisord...") + partition.generateSupervisorConfiguration() + partition.addServiceToCustomGroup(group_name, + wipe_wrapper_list, + wipe_base_folder) + + partition.writeSupervisorConfigurationFile() + + # check the state of all process, if the process is not started yes, start it + supervisord = partition.getSupervisorRPC() + process_list_string = "" + for name in wipe_wrapper_list: + process_name = group_name + ':' + name + process_list_string += process_name + '\n' + status = supervisord.getProcessInfo(process_name) + if status['start'] == 0: + # process is not started yet + logger.info("Starting pre-destroy process %r..." % name) + supervisord.startProcess(process_name, False) + + # ask to slapgrid to check theses scripts before destroy partition + with open(wait_filepath, 'w') as f: + f.write(process_list_string) diff --git a/slapos/tests/slapgrid.py b/slapos/tests/slapgrid.py index d2fec5385ea1a52878bf82522a3527153f962e21..68b398ea466c59f48564744944ee596f3f7dd164 100644 --- a/slapos/tests/slapgrid.py +++ b/slapos/tests/slapgrid.py @@ -2414,3 +2414,70 @@ class TestSlapgridCPWithTransaction(MasterMixin, unittest.TestCase): self.assertInstanceDirectoryListEqual(['0']) self.assertFalse(os.path.exists(request_list_file)) + +class TestSlapgridCPWithPreDeleteScript(MasterMixin, unittest.TestCase): + + def test_one_partition_pre_destroy_service(self): + from slapos import manager as slapmanager + from slapos.manager.predestroy import WIPE_WRAPPER_BASE_PATH + computer = ComputerForTest(self.software_root, self.instance_root) + with httmock.HTTMock(computer.request_handler): + partition = computer.instance_list[0] + pre_delete_dir = os.path.join(partition.partition_path, WIPE_WRAPPER_BASE_PATH) + pre_delete_script = os.path.join(pre_delete_dir, 'slapos_pre_delete') + partition.requested_state = 'started' + partition.software.setBuildout(WRAPPER_CONTENT) + self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS) + os.makedirs(pre_delete_dir, 0o700) + with open(pre_delete_script, 'w') as f: + f.write("""#!/bin/sh + +echo "Running script to wipe this partition..." + +for i in {1..3} +do + echo "sleeping for 1s..." + sleep 1 +done + +echo "finished wipe disk." + +exit 0 +""") + os.chmod(pre_delete_script, 0754) + self.assertInstanceDirectoryListEqual(['0']) + self.assertItemsEqual(os.listdir(partition.partition_path), + ['.slapgrid', '.0_wrapper.log', 'buildout.cfg', 'var', + 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay']) + wrapper_log = os.path.join(partition.partition_path, '.0_wrapper.log') + self.assertLogContent(wrapper_log, 'Working') + self.assertItemsEqual(os.listdir(self.software_root), [partition.software.software_hash]) + self.assertEqual(computer.sequence, + ['/getFullComputerInformation', '/availableComputerPartition', + '/startedComputerPartition']) + self.assertEqual(partition.state, 'started') + partition.requested_state = 'stopped' + self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS) + self.assertEqual(partition.state, 'stopped') + manager_list = slapmanager.from_config({'manager_list': 'predestroy'}) + self.grid._manager_list = manager_list + + partition.requested_state = 'destroyed' + self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) + # Assert partition directory is not destroyed (pre-destroy is running) + self.assertInstanceDirectoryListEqual(['0']) + self.assertItemsEqual(os.listdir(partition.partition_path), + ['.slapgrid', '.0_wrapper.log', 'buildout.cfg', 'var', + 'etc', 'software_release', 'worked', '.slapos-retention-lock-delay', + '.0-destroy_slapos_pre_delete.log', '.slapos-wait-services', + '.slapos-request-transaction-0']) + self.assertItemsEqual(os.listdir(self.software_root), + [partition.software.software_hash]) + + # wait until the pre-destroy script is finished + time.sleep(5) + + self.assertEqual(self.grid.agregateAndSendUsage(), slapgrid.SLAPGRID_SUCCESS) + # Assert partition directory is empty + self.assertInstanceDirectoryListEqual(['0']) + self.assertItemsEqual(os.listdir(partition.partition_path), [])