slapgrid: Start services even without connection to master

See merge request !515

slapos/cli/boot.py

@@ -31,7 +31,6 @@ from __future__ import print_function
 
 import subprocess
 from six.moves.urllib.parse import urlparse
-from six.moves import xmlrpc_client as xmlrpclib
 from time import sleep
 import glob
 import os
@@ -41,12 +40,7 @@ from netaddr import valid_ipv4, valid_ipv6
 from slapos.cli.command import check_root_user
 from slapos.cli.entry import SlapOSApp
 from slapos.cli.config import ConfigCommand
-from slapos.format import isGlobalScopeAddress
-from slapos.grid.slapgrid import (COMPUTER_PARTITION_REQUESTED_STATE_FILENAME,
-                                  COMPUTER_PARTITION_STARTED_STATE)
-from slapos.grid.svcbackend import  (_getSupervisordSocketPath,
-                                     getSupervisorRPC,
-                                     launchSupervisord)
+from slapos.format import isGlobalScopeAddress, FormatReturn
 from slapos.util import string_to_boolean
 import argparse
 import logging
@@ -65,58 +59,13 @@ def _removeTimestamp(instancehome, partition_base_name):
        logger.info("Removing %s", timestamp_path)
        os.remove(timestamp_path)
 
-def _startComputerPartition(partition_id, supervisord_socket):
-    """
-    With supervisord, start the instance that was deployed
-    """
-    try:
-      with getSupervisorRPC(supervisord_socket) as supervisor:
-        supervisor.startProcessGroup(partition_id, False)
-    except xmlrpclib.Fault as exc:
-      if exc.faultString.startswith('BAD_NAME:'):
-        logger.info("Nothing to start on %s...", partition_id)
-      else:
-        raise
-    else:
-      logger.info("Requested start of %s...", partition_id)
-
-def _startComputerPartitionList(instance_root, partition_base_name):
-  """
-  Start services for partition which has requested state to 'started'
-  """
-  partition_glob_path = os.path.join(
-        instance_root,
-        "%s*" % partition_base_name)
-  launchSupervisord(instance_root=instance_root, logger=logger)
-  for partition_path in glob.glob(partition_glob_path):
-    partition_state_path = os.path.join(
-        partition_path,
-        COMPUTER_PARTITION_REQUESTED_STATE_FILENAME
-    )
-    supervisord_socket_path = _getSupervisordSocketPath(
-      instance_root,
-      logger
-    )
-    if os.path.exists(partition_state_path):
-      partition_state = ""
-      with open(partition_state_path) as f:
-        partition_state = f.read()
-      if partition_state == COMPUTER_PARTITION_STARTED_STATE:
-        # Call start for this computer partition
-        _startComputerPartition(
-          os.path.basename(partition_path.rstrip('/')),
-          supervisord_socket_path
-        )
 
 def _runBang(app):
     """
     Launch slapos node format.
     """
     logger.info("[BOOT] Invoking slapos node bang...")
-    result = app.run(['node', 'bang', '-m', 'Reboot'])
-    if result == 1:
-      return 0
-    return 1
+    return app.run(['node', 'bang', '-m', 'Reboot'])
 
 
 def _runFormat(app):
@@ -124,12 +73,7 @@ def _runFormat(app):
     Launch slapos node format.
     """
     logger.info("[BOOT] Invoking slapos node format...")
-    # '--local' parameter is to prevent node format command to post data to
-    # master, so this command can work without internet and setup partitions IP.
-    result = app.run(['node', 'format', '--now', '--local', '--verbose'])
-    if result == 1:
-      return 0
-    return 1
+    return app.run(['node', 'format', '--now', '--verbose'])
 
 
 def _ping(hostname):
@@ -189,6 +133,16 @@ def _ping_hostname(hostname):
       is_ready = _ping6(hostname)
 
 
+def _ping_master(master_hostname):
+  if valid_ipv4(master_hostname):
+    _test_ping(master_hostname)
+  elif valid_ipv6(master_hostname):
+    _test_ping6(master_hostname)
+  else:
+    # hostname
+    _ping_hostname(master_hostname)
+
+
 def _waitIpv6Ready(ipv6_interface):
   """
     test if ipv6 is ready on ipv6_interface
@@ -204,6 +158,7 @@ def _waitIpv6Ready(ipv6_interface):
           "try again in 5 seconds...", ipv6_interface)
     sleep(5)
 
+
 class BootCommand(ConfigCommand):
     """
     Test network and invoke simple format and bang (Use on Linux startup)
@@ -247,25 +202,28 @@ class BootCommand(ConfigCommand):
             _waitIpv6Ready(ipv6_interface)
 
         app = SlapOSApp()
-        # Make sure slapos node format returns ok
-        while not _runFormat(app):
-            logger.error("[BOOT] Fail to format, try again in 15 seconds...")
-            sleep(15)
-
-        # Start computer partition services
-        _startComputerPartitionList(instance_root, partition_base_name)
-
-        # Check that node can ping master
-        if valid_ipv4(master_hostname):
-            _test_ping(master_hostname)
-        elif valid_ipv6(master_hostname):
-            _test_ping6(master_hostname)
-        else:
-            # hostname
-            _ping_hostname(master_hostname)
+        while True:
+            # Make sure slapos node format returns ok
+            result = _runFormat(app)
+
+            if result == FormatReturn.FAILURE:
+                logger.error("[BOOT] Fail to format, try again in 15 seconds...")
+                sleep(15)
+                continue
+
+            if result == FormatReturn.OFFLINE_SUCCESS:
+                logger.error(
+                    "[BOOT] Fail to post format information"
+                    ", try again when connection to master is up..."
+                )
+                sleep(15)
+                _ping_master(master_hostname)
+                continue
+
+            break
 
         # Make sure slapos node bang returns ok
-        while not _runBang(app):
+        while _runBang(app):
             logger.error("[BOOT] Fail to bang, try again in 15 seconds...")
             sleep(15)
 

slapos/cli/format.py

@@ -83,12 +83,6 @@ class FormatCommand(ConfigCommand):
                         help='Launch slapformat without delay'
                              ' (default: %(default)s)')
 
-        ap.add_argument('--local',
-                        default=False, # can have a default as it is not in .cfg
-                        action="store_true",
-                        help='Keep format data locally, do not post xml to master'
-                             ' (default: %(default)s)')
-
         ap.add_argument('-n', '--dry_run',
                         default=False, # can have a default as it is not in .cfg
                         action="store_true",
@@ -131,4 +125,4 @@ class FormatCommand(ConfigCommand):
 
         tracing_monkeypatch(conf)
 
-        do_format(conf=conf)
+        return do_format(conf=conf)

slapos/format.py

@@ -30,6 +30,7 @@
 
 from six.moves import configparser
 import distro
+import enum
 import errno
 import fcntl
 import grp
@@ -68,6 +69,12 @@ from slapos import version
 from slapos import manager as slapmanager
 
 
+class FormatReturn(enum.IntEnum):
+  SUCCESS = 0
+  FAILURE = 1
+  OFFLINE_SUCCESS = 2
+
+
 logger = logging.getLogger("slapos.format")
 
 
@@ -1578,40 +1585,50 @@ def random_delay(conf):
 
 
 def do_format(conf):
-  random_delay(conf)
+  try:
+    random_delay(conf)
 
-  if conf.input_definition_file:
-    computer = parse_computer_definition(conf, conf.input_definition_file)
-  else:
-    # no definition file, figure out computer
-    computer = parse_computer_xml(conf, conf.computer_xml)
-
-  computer.instance_storage_home = conf.instance_storage_home
-  conf.logger.info('Updating computer')
-  address = computer.getAddress()
-  computer.address = address['addr']
-  computer.netmask = address['netmask']
-
-  if conf.output_definition_file:
-    write_computer_definition(conf, computer)
-
-  computer.format(alter_user=conf.alter_user,
-                     alter_network=conf.alter_network,
-                     create_tap=conf.create_tap)
-
-  if getattr(conf, 'certificate_repository_path', None):
-    mkdir_p(conf.certificate_repository_path, mode=0o700)
-
-  computer.update()
-  # Dumping and sending to the erp5 the current configuration
-  if not conf.dry_run:
-    computer.dump(path_to_xml=conf.computer_xml,
-                  path_to_json=conf.computer_json,
-                  logger=conf.logger)
-  if not conf.local:
+    if conf.input_definition_file:
+      computer = parse_computer_definition(conf, conf.input_definition_file)
+    else:
+      # no definition file, figure out computer
+      computer = parse_computer_xml(conf, conf.computer_xml)
+
+    computer.instance_storage_home = conf.instance_storage_home
+    conf.logger.info('Updating computer')
+    address = computer.getAddress()
+    computer.address = address['addr']
+    computer.netmask = address['netmask']
+
+    if conf.output_definition_file:
+      write_computer_definition(conf, computer)
+
+    computer.format(alter_user=conf.alter_user,
+                      alter_network=conf.alter_network,
+                      create_tap=conf.create_tap)
+
+    if getattr(conf, 'certificate_repository_path', None):
+      mkdir_p(conf.certificate_repository_path, mode=0o700)
+
+    computer.update()
+    # Dumping and sending to the erp5 the current configuration
+    if not conf.dry_run:
+      computer.dump(path_to_xml=conf.computer_xml,
+                    path_to_json=conf.computer_json,
+                    logger=conf.logger)
     conf.logger.info('Posting information to %r' % conf.master_url)
-    computer.send(conf)
-  conf.logger.info('slapos successfully prepared the computer.')
+    try:
+      computer.send(conf)
+      return FormatReturn.SUCCESS
+    except Exception:
+      conf.logger.exception('failed to transfer information to %r' % conf.master_url)
+      return FormatReturn.OFFLINE_SUCCESS
+    finally:
+      conf.logger.info('slapos successfully prepared the computer.')
+  except Exception:
+    conf.logger.exception('slapos failed to prepare the computer.')
+    return FormatReturn.FAILURE
+
 
 
 class FormatConfig(object):

slapos/grid/SlapObject.py

@@ -39,6 +39,9 @@ import subprocess
 import tarfile
 import tempfile
 import time
+
+from collections import defaultdict
+
 from six.moves import xmlrpc_client as xmlrpclib, range
 from six.moves.configparser import ConfigParser
 
@@ -64,8 +67,13 @@ REQUIRED_COMPUTER_PARTITION_PERMISSION = 0o750
 CP_STORAGE_FOLDER_NAME = 'DATA'
 
 # XXX not very clean. this is changed when testing
-PROGRAM_PARTITION_TEMPLATE = bytes2str(pkg_resources.resource_string(__name__,
-            'templates/program_partition_supervisord.conf.in'))
+PROGRAM_PARTITION_TEMPLATE = bytes2str(
+  pkg_resources.resource_string(
+    __name__, 'templates/program_partition_supervisord.conf.in'))
+
+GROUP_PARTITION_TEMPLATE = bytes2str(
+  pkg_resources.resource_string(
+    __name__, 'templates/group_partition_supervisord.conf.in'))
 
 
 def free_space(path, fn):
@@ -409,7 +417,7 @@ class Partition(object):
                software_path,
                instance_path,
                shared_part_list,
-               supervisord_partition_configuration_path,
+               supervisord_partition_configuration_dir,
                supervisord_socket,
                computer_partition,
                computer_id,
@@ -436,8 +444,8 @@ class Partition(object):
     self.run_path = os.path.join(self.instance_path, 'etc', 'run')
     self.service_path = os.path.join(self.instance_path, 'etc', 'service')
     self.prerm_path = os.path.join(self.instance_path, 'etc', 'prerm')
-    self.supervisord_partition_configuration_path = \
-        supervisord_partition_configuration_path
+    self.supervisord_partition_configuration_dir = \
+        supervisord_partition_configuration_dir
     self.supervisord_socket = supervisord_socket
     self.computer_partition = computer_partition
     self.computer_id = computer_id
@@ -505,7 +513,8 @@ class Partition(object):
       new_content = partition_certificate[name]
       old_content = None
       if os.path.exists(path):
-        old_content = open(path).read()
+        with open(path) as f:
+          old_content = f.read()
 
       if old_content != new_content:
         if old_content is None:
@@ -524,53 +533,24 @@ class Partition(object):
     gid = stat_info.st_gid
     return (uid, gid)
 
-  def addProgramToGroup(self, partition_id, program_id, name, command,
-                        as_user=True):
-    if as_user:
-      uid, gid = self.getUserGroupId()
-    else:
-      uid, gid = 0, 0
-    self.partition_supervisor_configuration += '\n' + \
-      PROGRAM_PARTITION_TEMPLATE % {
-        'program_id': '{}_{}'.format(partition_id, program_id),
-        'program_directory': self.instance_path,
-        'program_command': command,
-        'program_name': name,
-        'instance_path': self.instance_path,
-        'user_id': uid,
-        'group_id': gid,
-        # As supervisord has no environment to inherit, setup a minimalistic one
-        'HOME': pwd.getpwuid(uid).pw_dir,
-        'USER': pwd.getpwuid(uid).pw_name,
-      }
-
-  def addCustomGroup(self, group_suffix, partition_id, program_list):
-    group_partition_template = bytes2str(pkg_resources.resource_string(__name__,
-      'templates/group_partition_supervisord.conf.in'))
-    group_id = '{}-{}'.format(partition_id, group_suffix)
+  def getGroupIdFromSuffix(self, suffix=None):
+    partition_id = self.partition_id
+    return '%s-%s' % (partition_id, suffix) if suffix else partition_id
 
-    self.supervisor_configuration_group += group_partition_template % {
-      'instance_id': group_id,
-      'program_list': ','.join(['{}_{}'.format(group_id, program_id)
-                                for program_id in program_list]),
-    }
+  def addProgramToGroup(self, suffix, program_id, name, command, as_user=True):
+    group = self.getGroupIdFromSuffix(suffix)
+    self.supervisor_conf[group][program_id] = (name, command, as_user)
 
-    return group_id
+  def addServicesToGroup(self, runner_list, path, extension=''):
+    self.addServicesToCustomGroup(None, runner_list, path, extension)
 
-  def addServiceToGroup(self, partition_id, runner_list, path, extension=''):
+  def addServicesToCustomGroup(self, suffix, runner_list, path, extension=''):
+    """Add new services to supervisord that belong to specific group"""
     for runner in runner_list:
       program_id = runner
       program_name = runner + extension
       program_command = os.path.join(path, runner)
-      self.addProgramToGroup(partition_id, program_id, program_name,
-                             program_command)
-
-  def addServiceToCustomGroup(self, group_suffix, partition_id, runner_list,
-      path, extension=''):
-    """Add new services to supervisord that belong to specific group"""
-    group_id = self.addCustomGroup(group_suffix, partition_id,
-                                   runner_list)
-    return self.addServiceToGroup(group_id, runner_list, path, extension)
+      self.addProgramToGroup(suffix, program_id, program_name, program_command)
 
   def updateSymlink(self, sr_symlink, software_path):
     if os.path.lexists(sr_symlink):
@@ -586,7 +566,7 @@ class Partition(object):
     installs the software partition with the help of buildout
     """
     self.logger.info("Installing Computer Partition %s..."
-        % self.computer_partition.getId())
+        % self.partition_id)
 
     self.check_free_space()
 
@@ -708,7 +688,6 @@ class Partition(object):
                          logger=self.logger,
                          debug=self.buildout_debug,
                          timeout=self.partition_timeout)
-    self.generateSupervisorConfigurationFile()
     self.createRetentionLockDelay()
     self.instance_python = getPythonExecutableFromSoftwarePath(self.software_path)
 
@@ -723,8 +702,7 @@ class Partition(object):
     """
     runner_list = []
     service_list = []
-    self.partition_supervisor_configuration = ""
-    self.supervisor_configuration_group = ""
+    self.supervisor_conf = defaultdict(dict)
     if os.path.exists(self.run_path):
       if os.path.isdir(self.run_path):
         runner_list = sorted(os.listdir(self.run_path))
@@ -734,76 +712,111 @@ class Partition(object):
     if len(runner_list) == 0 and len(service_list) == 0:
       self.logger.warning('No runners nor services found for partition %r' %
           self.partition_id)
-      if os.path.exists(self.supervisord_partition_configuration_path):
-        os.unlink(self.supervisord_partition_configuration_path)
     else:
-      partition_id = self.computer_partition.getId()
-      group_partition_template = bytes2str(pkg_resources.resource_string(__name__,
-          'templates/group_partition_supervisord.conf.in'))
-      self.supervisor_configuration_group = group_partition_template % {
-          'instance_id': partition_id,
-          'program_list': ','.join(['_'.join([partition_id, runner])
-                                    for runner in runner_list + service_list])
-      }
       # Same method to add to service and run
-      self.addServiceToGroup(partition_id, runner_list, self.run_path)
-      self.addServiceToGroup(partition_id, service_list, self.service_path,
-                             extension=WATCHDOG_MARK)
+      self.addServicesToGroup(runner_list, self.run_path)
+      self.addServicesToGroup(
+        service_list, self.service_path, extension=WATCHDOG_MARK)
+
+  def writeSupervisorConfigurationFiles(self):
+    """
+      Write supervisord configuration files and update supervisord
+    """
+    remaining = set(
+      f for f in os.listdir(self.supervisord_partition_configuration_dir)
+      if f.startswith(self.partition_id)
+    )
+    for group, programs in self.supervisor_conf.items():
+      filename = '%s.conf' % group
+      filepath = os.path.join(
+        self.supervisord_partition_configuration_dir, filename)
+      supervisor_conf = GROUP_PARTITION_TEMPLATE % {
+        'instance_id': group,
+        'program_list': ','.join('%s_%s' % (group, p) for p in programs.keys()),
+      }
+      for program_id, (name, command, as_user) in programs.items():
+        uid, gid = self.getUserGroupId() if as_user else (0, 0)
+        supervisor_conf += '\n' + \
+          PROGRAM_PARTITION_TEMPLATE % {
+            'program_id': '%s_%s' % (group, program_id),
+            'program_directory': self.instance_path,
+            'program_command': command,
+            'program_name': name,
+            'instance_path': self.instance_path,
+            'user_id': uid,
+            'group_id': gid,
+            # As supervisord has no environment to inherit, setup a minimalistic one
+            'HOME': pwd.getpwuid(uid).pw_dir,
+            'USER': pwd.getpwuid(uid).pw_name,
+          }
+      remaining.discard(filename)
+      updateFile(filepath, supervisor_conf)
+    for filename in remaining:
+      filepath = os.path.join(
+        self.supervisord_partition_configuration_dir, filename)
+      os.unlink(filepath)
+    if self.supervisor_conf or remaining:
+      self.updateSupervisor()
 
-  def writeSupervisorConfigurationFile(self):
+  def removeSupervisorConfigurationFiles(self):
     """
-      Write supervisord configuration file and update supervisord
+      Remove supervisord configuration files if any exist and update supervisord
     """
-    if self.supervisor_configuration_group and \
-        self.partition_supervisor_configuration:
-      updateFile(self.supervisord_partition_configuration_path,
-                 self.supervisor_configuration_group +
-                 self.partition_supervisor_configuration)
-    self.updateSupervisor()
-
-  def generateSupervisorConfigurationFile(self):
+    filenames = [
+      f for f in os.listdir(self.supervisord_partition_configuration_dir)
+      if f.startswith(self.partition_id)
+    ]
+    for filename in filenames:
+      filepath = os.path.join(
+        self.supervisord_partition_configuration_dir, filename)
+      os.unlink(filepath)
+    if filenames:
+      self.updateSupervisor()
+
+  def updateSupervisorConfiguration(self):
     """
       update supervisord with new processes
     """
     self.generateSupervisorConfiguration()
-    self.writeSupervisorConfigurationFile()
+    self.writeSupervisorConfigurationFiles()
 
   def start(self):
     """Asks supervisord to start the instance. If this instance is not
     installed, we install it.
     """
-    partition_id = self.computer_partition.getId()
+    self.updateSupervisorConfiguration()
+    partition_id = self.partition_id
     try:
       with self.getSupervisorRPC() as supervisor:
         supervisor.startProcessGroup(partition_id, False)
     except xmlrpclib.Fault as exc:
       if exc.faultString.startswith('BAD_NAME:'):
-        self.logger.info("Nothing to start on %s..." %
-                         self.computer_partition.getId())
+        self.logger.info("Nothing to start on %s..." % partition_id)
       else:
         raise
     else:
-      self.logger.info("Requested start of %s..." % self.computer_partition.getId())
+      self.logger.info("Requested start of %s..." % partition_id)
 
   def stop(self):
     """Asks supervisord to stop the instance."""
-    partition_id = self.computer_partition.getId()
+    partition_id = self.partition_id
+    filename = partition_id + '.conf'
+    filepath = os.path.join(
+      self.supervisord_partition_configuration_dir, filename)
     try:
-      with self.getSupervisorRPC() as supervisor:
-        supervisor.stopProcessGroup(partition_id, False)
-    except xmlrpclib.Fault as exc:
-      if exc.faultString.startswith('BAD_NAME:'):
-        self.logger.info('Partition %s not known in supervisord, ignoring' % partition_id)
-      else:
+      os.unlink(filepath)
+    except OSError as e:
+      if e.errno != errno.ENOENT:
         raise
     else:
-      self.logger.info("Requested stop of %s..." % self.computer_partition.getId())
+      self.updateSupervisor()
+      self.logger.info("Requested stop of %s..." % partition_id)
 
   def destroy(self):
     """Destroys the partition and makes it available for subsequent use."
     """
     self.logger.info("Destroying Computer Partition %s..."
-        % self.computer_partition.getId())
+        % self.partition_id)
 
     self.createRetentionLockDate()
     if not self.checkRetentionIsAuthorized():
@@ -856,9 +869,7 @@ class Partition(object):
       # Cleanup all Data storage location of this partition
       
 
-      if os.path.exists(self.supervisord_partition_configuration_path):
-        os.remove(self.supervisord_partition_configuration_path)
-      self.updateSupervisor()
+      self.removeSupervisorConfigurationFiles()
     except IOError as exc:
       raise IOError("I/O error while freeing partition (%s): %s" % (self.instance_path, exc))
 

slapos/grid/slapgrid.py

@@ -54,6 +54,8 @@ if sys.version_info < (2, 6):
   warnings.warn('Used python version (%s) is old and has problems with'
       ' IPv6 connections' % sys.version.split('\n')[0])
 
+from requests.exceptions import RequestException
+
 from lxml import etree
 
 from slapos import manager as slapmanager
@@ -70,7 +72,8 @@ from slapos.grid.SlapObject import Software, Partition
 from slapos.grid.svcbackend import (launchSupervisord,
                                     createSupervisordConfiguration,
                                     _getSupervisordConfigurationDirectory,
-                                    _getSupervisordSocketPath)
+                                    _getSupervisordSocketPath,
+                                    getSupervisorRPC)
 from slapos.grid.utils import (md5digest,
                               dropPrivileges,
                               SlapPopen,
@@ -92,10 +95,10 @@ COMPUTER_PARTITION_STOPPED_STATE = 'stopped'
 SLAPGRID_SUCCESS = 0
 SLAPGRID_FAIL = 1
 SLAPGRID_PROMISE_FAIL = 2
+SLAPGRID_OFFLINE_SUCCESS = 3
 PROMISE_TIMEOUT = 20
 
 COMPUTER_PARTITION_TIMESTAMP_FILENAME = '.timestamp'
-COMPUTER_PARTITION_REQUESTED_STATE_FILENAME = '.requested_state'
 COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME = '.slapos_latest_bang_timestamp'
 COMPUTER_PARTITION_INSTALL_ERROR_FILENAME = '.slapgrid-%s-error.log'
 COMPUTER_PARTITION_WAIT_LIST_FILENAME = '.slapos-report-wait-service-list'
@@ -1081,9 +1084,8 @@ stderr_logfile_backups=1
       software_path=software_path,
       instance_path=instance_path,
       shared_part_list=self.shared_part_list,
-      supervisord_partition_configuration_path=os.path.join(
-        _getSupervisordConfigurationDirectory(self.instance_root),
-        computer_partition_id + '.conf'),
+      supervisord_partition_configuration_dir=(
+        _getSupervisordConfigurationDirectory(self.instance_root)),
       supervisord_socket=self.supervisord_socket,
       computer_partition=computer_partition,
       computer_id=self.computer_id,
@@ -1137,10 +1139,6 @@ stderr_logfile_backups=1
         instance_path,
         COMPUTER_PARTITION_TIMESTAMP_FILENAME
     )
-    partition_state_path = os.path.join(
-        instance_path,
-        COMPUTER_PARTITION_REQUESTED_STATE_FILENAME
-    )
     parameter_dict = computer_partition.getInstanceParameterDict()
     timestamp = parameter_dict.get('timestamp')
 
@@ -1178,9 +1176,8 @@ stderr_logfile_backups=1
       software_path=software_path,
       instance_path=instance_path,
       shared_part_list=self.shared_part_list,
-      supervisord_partition_configuration_path=os.path.join(
-        _getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' %
-        computer_partition_id),
+      supervisord_partition_configuration_dir=(
+        _getSupervisordConfigurationDirectory(self.instance_root)),
       supervisord_socket=self.supervisord_socket,
       computer_partition=computer_partition,
       computer_id=self.computer_id,
@@ -1243,7 +1240,6 @@ stderr_logfile_backups=1
 
               return
         os.remove(timestamp_path)
-        os.remove(partition_state_path)
 
     # Include Partition Logging
     log_folder_path = "%s/.slapgrid/log" % instance_path
@@ -1358,8 +1354,6 @@ stderr_logfile_backups=1
     if timestamp:
       with open(timestamp_path, 'w') as f:
         f.write(str(timestamp))
-      with open(partition_state_path, 'w') as f:
-        f.write(str(computer_partition_state))
 
   def FilterComputerPartitionList(self, computer_partition_list):
     """
@@ -1429,6 +1423,12 @@ stderr_logfile_backups=1
     return filtered_computer_partition_list
 
   def processComputerPartitionList(self):
+    try:
+      return self.processComputerPartitionListOnline()
+    except RequestException:
+      return self.processComputerPartitionListOffline()
+
+  def processComputerPartitionListOnline(self):
     """
     Will start supervisord and process each Computer Partition.
     """
@@ -1455,6 +1455,10 @@ stderr_logfile_backups=1
         # Process the partition itself
         self.processComputerPartition(computer_partition)
 
+      # Handle connection loss at the next level
+      except RequestException:
+        raise
+
       # Send log before exiting
       except (SystemExit, KeyboardInterrupt):
         computer_partition.error(traceback.format_exc(), logger=self.logger)
@@ -1511,6 +1515,20 @@ stderr_logfile_backups=1
       return SLAPGRID_PROMISE_FAIL
     return SLAPGRID_SUCCESS
 
+  def processComputerPartitionListOffline(self):
+    self.logger.info('Processing computer partitions offline...')
+    try:
+      supervisord_socket_path = _getSupervisordSocketPath(
+        self.instance_root,
+        self.logger
+      )
+      with getSupervisorRPC(supervisord_socket_path) as supervisor:
+        supervisor.startAllProcesses(False)
+    except Exception:
+      self.logger.exception('Error in offline mode while starting partitions:')
+      return SLAPGRID_FAIL
+    return SLAPGRID_OFFLINE_SUCCESS
+
   def processPromiseList(self):
     """
     Will check and process promises for each Computer Partition.
@@ -1841,9 +1859,8 @@ stderr_logfile_backups=1
             instance_path=os.path.join(self.instance_root,
                 computer_partition.getId()),
             shared_part_list=self.shared_part_list,
-            supervisord_partition_configuration_path=os.path.join(
-              _getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' %
-              computer_partition_id),
+            supervisord_partition_configuration_dir=(
+              _getSupervisordConfigurationDirectory(self.instance_root)),
             supervisord_socket=self.supervisord_socket,
             computer_partition=computer_partition,
             computer_id=self.computer_id,

slapos/manager/portredir.py

@@ -156,17 +156,15 @@ class Manager(object):
     # Generate supervisord configuration with socat processes added
     partition.generateSupervisorConfiguration()
 
-    group_id = partition.addCustomGroup('socat', partition.partition_id,
-                                        [program['name']
-                                         for program in socat_programs])
     for program in socat_programs:
-      partition.addProgramToGroup(group_id, program['name'], program['name'],
+      partition.addProgramToGroup('socat', program['name'], program['name'],
                                   program['command'],
                                   as_user=program['as_user'])
 
-    partition.writeSupervisorConfigurationFile()
+    partition.writeSupervisorConfigurationFiles()
 
     # Start processes
+    group_id = partition.getGroupIdFromSuffix('socat')
     with partition.getSupervisorRPC() as supervisor:
       for program in socat_programs:
         process_name = '{}:{}'.format(group_id, program['name'])

slapos/manager/prerm.py

@@ -73,11 +73,9 @@ class Manager(object):
       group_suffix = "prerm"
       logger.info("Adding pre-delete scripts to supervisord...")
       partition.generateSupervisorConfiguration()
-      partition.addServiceToCustomGroup(group_suffix,
-                                        partition_id,
-                                        wrapper_list,
-                                        partition.prerm_path)
-      partition.writeSupervisorConfigurationFile()
+      partition.addServicesToCustomGroup(
+        group_suffix, wrapper_list, partition.prerm_path)
+      partition.writeSupervisorConfigurationFiles()
 
       # check the state of all process, if the process is not started yes, start it
       with partition.getSupervisorRPC() as supervisor:

slapos/slap/standalone.py

@@ -737,11 +737,7 @@ class StandaloneSlapOS(object):
         state=state)
 
   def start(self):
-    """Start the system.
-
-    If system was stopped, it will start partitions.
-    If system was already running, this does not restart partitions.
-    """
+    """Start the system."""
     self._logger.debug("Starting StandaloneSlapOS in %s", self._base_directory)
     self._ensureSupervisordStarted()
     self._ensureSlapOSAvailable()

slapos/tests/test_cli.py

@@ -416,13 +416,8 @@ class TestCliBoot(CliMixin):
     os.mkdir(os.path.join(instance_root, partition_base_name + '1'))
     timestamp = os.path.join(
         instance_root, partition_base_name + '1', '.timestamp')
-    requested_state_path = os.path.join(instance_root,
-                                        partition_base_name + '1',
-                                        '.requested_state')
     with open(timestamp, 'w') as f:
       f.write("1578552471")
-    with open(requested_state_path, 'w') as f:
-      f.write("started")
 
     # make a config file using this instance root
     with tempfile.NamedTemporaryFile(mode='w') as slapos_conf:
@@ -441,27 +436,24 @@ class TestCliBoot(CliMixin):
 
       # run slapos node boot
       app = slapos.cli.entry.SlapOSApp()
+      fake = mock.Mock(return_value=mock.Mock(**{'run.return_value': 0}))
       with patch('slapos.cli.boot.check_root_user', return_value=True) as check_root_user,\
-          patch('slapos.cli.boot.SlapOSApp') as SlapOSApp,\
+          patch('slapos.cli.boot.SlapOSApp', new=fake) as SlapOSApp,\
           patch('slapos.cli.boot.ConfigCommand.config_path', return_value=slapos_conf.name), \
           patch(
               'slapos.cli.boot.netifaces.ifaddresses',
               return_value={socket.AF_INET6: ({'addr': '2000::1'},),},) as ifaddresses,\
-          patch('slapos.cli.boot._startComputerPartition', return_value=None) as start_partition,\
-          patch('slapos.cli.boot.launchSupervisord', return_value=None),\
           patch('slapos.cli.boot._ping_hostname', return_value=1) as _ping_hostname:
         app.run(('node', 'boot'))
 
       # boot command runs as root
       check_root_user.assert_called_once()
-      # Computer partition was started during boot
-      start_partition.assert_called_once()
       # it waits for interface to have an IPv6 address
       ifaddresses.assert_called_once_with('interface_name_from_config')
       # then ping master hostname to wait for connectivity
       _ping_hostname.assert_called_once_with('slap.vifib.com')
       # then format and bang
-      SlapOSApp().run.assert_any_call(['node', 'format', '--now', '--local', '--verbose'])
+      SlapOSApp().run.assert_any_call(['node', 'format', '--now', '--verbose'])
       SlapOSApp().run.assert_any_call(['node', 'bang', '-m', 'Reboot'])
 
       # timestamp files have been removed
@@ -483,17 +475,15 @@ class TestCliBoot(CliMixin):
         patch('slapos.cli.boot.netifaces.ifaddresses',
               side_effect=[net1, net2, net3]),\
         patch('slapos.cli.boot._ping_hostname', return_value=0),\
-        patch('slapos.cli.boot._startComputerPartitionList', return_value=None) as start_partition,\
         patch('slapos.cli.format.check_root_user', return_value=True),\
         patch('slapos.cli.format.logging.FileHandler', return_value=logging.NullHandler()),\
         patch('slapos.cli.bang.check_root_user', return_value=True),\
-        patch('slapos.cli.format.do_format', side_effect=[Exception, Exception, None]) as do_format,\
-        patch('slapos.cli.bang.do_bang', side_effect=[Exception, Exception, None]) as do_bang:
+        patch('slapos.cli.format.do_format', side_effect=[Exception, Exception, 0]) as do_format,\
+        patch('slapos.cli.bang.do_bang', side_effect=[Exception, Exception, 0]) as do_bang:
 
       app.run(('node', 'boot'))
 
     check_root_user.assert_called_once()
-    start_partition.assert_called_once()
 
     self.assertEqual(do_format.call_count, 3)
     self.assertEqual(do_bang.call_count, 3)

slapos/tests/test_object.py

@@ -87,7 +87,7 @@ original_upload_network_cached = networkcache.upload_network_cached
 originalBootstrapBuildout = utils.bootstrapBuildout
 originalLaunchBuildout = utils.launchBuildout
 originalUploadSoftwareRelease = Software.uploadSoftwareRelease
-originalPartitionGenerateSupervisorConfigurationFile = Partition.generateSupervisorConfigurationFile
+originalPartitionUpdateSupervisorConfiguration = Partition.updateSupervisorConfiguration
 
 class MasterMixin(BasicMixin, unittest.TestCase):
   """
@@ -170,8 +170,7 @@ class MasterMixin(BasicMixin, unittest.TestCase):
       software_path=software_path,
       instance_path=instance_path,
       shared_part_list=shared_part_list,
-      supervisord_partition_configuration_path=os.path.join(
-          supervisor_configuration_path, partition_id),
+      supervisord_partition_configuration_dir=supervisor_configuration_path,
       supervisord_socket=os.path.join(
           supervisor_configuration_path, 'supervisor.sock'),
       computer_partition=slap_computer_partition,
@@ -378,14 +377,14 @@ class TestPartitionSlapObject(MasterMixin, unittest.TestCase):
   def setUp(self):
     MasterMixin.setUp(self)
 
-    Partition.generateSupervisorConfigurationFile = FakeCallAndNoop()
+    Partition.updateSupervisorConfiguration = FakeCallAndNoop()
     utils.bootstrapBuildout = FakeCallAndNoop()
 
     utils.launchBuildout = FakeCallAndStore()
 
   def tearDown(self):
     MasterMixin.tearDown(self)
-    Partition.generateSupervisorConfigurationFile = originalPartitionGenerateSupervisorConfigurationFile
+    Partition.updateSupervisorConfiguration = originalPartitionUpdateSupervisorConfiguration
 
   def test_partition_timeout_default(self):
     software = self.createSoftware()
@@ -417,18 +416,6 @@ class TestPartitionSlapObject(MasterMixin, unittest.TestCase):
 
     self.assertTrue(utils.launchBuildout.called)
 
-  def test_instance_is_deploying_if_software_release_exists(self):
-    """
-    Test that slapgrid deploys an instance if its Software Release exists and
-    instance.cfg in the Software Release exists.
-    """
-    software = self.createSoftware()
-
-    partition = self.createPartition(software.url)
-    partition.install()
-
-    self.assertTrue(utils.launchBuildout.called)
-
   def test_backward_compatibility_instance_is_deploying_if_template_cfg_is_used(self):
     """
     Backward compatibility test, for old software releases.
@@ -507,50 +494,52 @@ class TestPartitionSupervisorConfig(MasterMixin, unittest.TestCase):
     utils.launchBuildout = FakeCallAndNoop()
 
   def test_grouped_program(self):
-    self.assertEqual(self.partition.supervisor_configuration_group, '')
-    self.assertEqual(self.partition.partition_supervisor_configuration, '')
+    self.partition.addProgramToGroup('test', 'sample-1', 'sample-1', '/bin/ls')
+    self.partition.writeSupervisorConfigurationFiles()
 
-    partition_id = self.partition.partition_id
+    group_id = self.partition.getGroupIdFromSuffix('test')
 
-    group_id = self.partition.addCustomGroup('test', partition_id,
-                                             ['sample-1'])
-
-    self.assertIn('group:{}-test'.format(partition_id),
-                  self.partition.supervisor_configuration_group)
+    filepath = os.path.join(
+      self.partition.supervisord_partition_configuration_dir,
+      group_id + '.conf'
+    )
 
-    self.partition.addProgramToGroup(group_id, 'sample-1', 'sample-1',
-                                     '/bin/ls')
+    with open(filepath) as f:
+      supervisor_conf = f.read()
 
-    self.assertIn('program:{}-test_sample-1'.format(partition_id),
-                  self.partition.partition_supervisor_configuration)
+    self.assertIn('group:' + group_id, supervisor_conf)
+    self.assertIn('program:%s_sample-1' % group_id, supervisor_conf)
 
   def test_simple_service(self):
-    self.assertEqual(self.partition.supervisor_configuration_group, '')
-    self.assertEqual(self.partition.partition_supervisor_configuration, '')
+    runners = ['runner-' + str(i) for i in range(3)]
+    path = os.path.join(self.partition.instance_path, 'etc/run')
+    self.partition.addServicesToGroup(runners, path)
+    self.partition.writeSupervisorConfigurationFiles()
 
-    partition_id = self.partition.partition_id
+    group_id = self.partition.getGroupIdFromSuffix()
 
-    runners = ['runner-{}'.format(i) for i in range(3)]
-    path = os.path.join(self.partition.instance_path, 'etc/run')
-    self.partition.addServiceToGroup(partition_id, runners, path)
+    filepath = os.path.join(
+      self.partition.supervisord_partition_configuration_dir,
+      group_id + '.conf'
+    )
+
+    with open(filepath) as f:
+      supervisor_conf = f.read()
 
     for i in range(3):
-      self.assertIn('program:{}_runner-{}'.format(partition_id, i),
-                    self.partition.partition_supervisor_configuration)
+      self.assertIn('program:%s_runner-%s' % (group_id, i), supervisor_conf)
 
-      runner_path = os.path.join(self.partition.instance_path, 'etc/run',
-                                 'runner-{}'.format(i))
 
 class TestPartitionDestructionLock(MasterMixin, unittest.TestCase):
   def setUp(self):
     MasterMixin.setUp(self)
-    Partition.generateSupervisorConfigurationFile = FakeCallAndNoop()
+    Partition.updateSupervisorConfiguration = FakeCallAndNoop()
     utils.bootstrapBuildout = FakeCallAndNoop()
     utils.launchBuildout = FakeCallAndStore()
 
   def tearDown(self):
     MasterMixin.tearDown(self)
-    Partition.generateSupervisorConfigurationFile = originalPartitionGenerateSupervisorConfigurationFile
+    Partition.updateSupervisorConfiguration = originalPartitionUpdateSupervisorConfiguration
 
   def test_retention_lock_delay_creation(self):
     delay = 42
@@ -640,13 +629,13 @@ class TestPartitionDestructionLock(MasterMixin, unittest.TestCase):
 class TestPartitionDestructionUnwritable(MasterMixin, unittest.TestCase):
   def setUp(self):
     MasterMixin.setUp(self)
-    Partition.generateSupervisorConfigurationFile = FakeCallAndNoop()
+    Partition.updateSupervisorConfiguration = FakeCallAndNoop()
     utils.bootstrapBuildout = FakeCallAndNoop()
     utils.launchBuildout = FakeCallAndStore()
 
   def tearDown(self):
     MasterMixin.tearDown(self)
-    Partition.generateSupervisorConfigurationFile = originalPartitionGenerateSupervisorConfigurationFile
+    Partition.updateSupervisorConfiguration = originalPartitionUpdateSupervisorConfiguration
 
   def test(self):
     software = self.createSoftware()

slapos/tests/test_slapgrid.py

@@ -388,7 +388,8 @@ class ComputerForTest(object):
                software_root,
                instance_root,
                instance_amount=1,
-               software_amount=1):
+               software_amount=1,
+               status_code=200):
     """
     Will set up instances, software and sequence
     """
@@ -397,6 +398,7 @@ class ComputerForTest(object):
     self.software_amount = software_amount
     self.software_root = software_root
     self.instance_root = instance_root
+    self.status_code = status_code
     self.ip_address_list = [
             ('interface1', '10.0.8.3'),
             ('interface2', '10.0.8.4'),
@@ -425,18 +427,18 @@ class ComputerForTest(object):
             and 'computer_id' in qs):
       slap_computer = self.getComputer(qs['computer_id'][0])
       return {
-              'status_code': 200,
+              'status_code': self.status_code,
               'content': dumps(slap_computer)
               }
     elif url.path == '/getHostingSubscriptionIpList':
       ip_address_list = self.ip_address_list
       return {
-              'status_code': 200,
+              'status_code': self.status_code,
               'content': dumps(ip_address_list)
               }
     elif url.path == '/getComputerPartitionCertificate':
       return {
-              'status_code': 200,
+              'status_code': self.status_code,
               'content': dumps({'certificate': 'SLAPOS_cert', 'key': 'SLAPOS_key'})
               }
     if req.method == 'POST' and 'computer_partition_id' in qs:
@@ -445,17 +447,17 @@ class ComputerForTest(object):
       instance.header_list.append(req.headers)
       if url.path == '/startedComputerPartition':
         instance.state = 'started'
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
       if url.path == '/stoppedComputerPartition':
         instance.state = 'stopped'
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
       if url.path == '/destroyedComputerPartition':
         instance.state = 'destroyed'
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
       if url.path == '/softwareInstanceBang':
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
       if url.path == "/updateComputerPartitionRelatedInstanceList":
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
       if url.path == '/softwareInstanceError':
         instance.error_log = '\n'.join(
             [
@@ -465,18 +467,18 @@ class ComputerForTest(object):
             ]
         )
         instance.error = True
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
 
     elif req.method == 'POST' and 'url' in qs:
       # XXX hardcoded to first software release!
       software = self.software_list[0]
       software.sequence.append(url.path)
       if url.path == '/availableSoftwareRelease':
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
       if url.path == '/buildingSoftwareRelease':
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
       if url.path == '/destroyedSoftwareRelease':
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
       if url.path == '/softwareReleaseError':
         software.error_log = '\n'.join(
             [
@@ -486,7 +488,7 @@ class ComputerForTest(object):
             ]
         )
         software.error = True
-        return {'status_code': 200}
+        return {'status_code': self.status_code}
 
     else:
       return {'status_code': 500}
@@ -1021,6 +1023,69 @@ exit 1
                         '/stoppedComputerPartition'])
       self.assertEqual('stopped', instance.state)
 
+  def test_one_partition_started_no_master(self):
+    computer = self.getTestComputerClass()(self.software_root, self.instance_root, status_code=503)
+    with httmock.HTTMock(computer.request_handler):
+      partition = computer.instance_list[0]
+      partition.requested_state = 'started'
+      partition.software.setBuildout()
+      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_OFFLINE_SUCCESS)
+      self.assertInstanceDirectoryListEqual(['0'])
+      six.assertCountEqual(self, os.listdir(partition.partition_path), []) # buildout hasn't run
+      six.assertCountEqual(self, os.listdir(self.software_root), [partition.software.software_hash])
+      self.assertEqual(computer.sequence, ['/getFullComputerInformation'])
+      self.assertEqual(partition.state, None)
+
+  def test_one_partition_started_after_master_connection_loss(self):
+    computer = self.getTestComputerClass()(self.software_root, self.instance_root)
+    partition = computer.instance_list[0]
+    partition.requested_state = 'started'
+    partition.software.setBuildout()
+    run_path = os.path.join(partition.partition_path, 'etc', 'run')
+    os.makedirs(run_path)
+    with open(os.path.join(run_path, 'runner'), 'w') as f:
+      f.write("#!/bin/sh\necho 'Working'\ntouch 'runner_worked'")
+      os.fchmod(f.fileno(), 0o755)
+
+    runner_worked_file = os.path.join(partition.partition_path, 'runner_worked')
+    def assertRunnerWorked():
+      for _ in range(50):
+        if os.path.exists(runner_worked_file):
+          break
+        time.sleep(0.1)
+      else:
+        self.assertTrue(os.path.exists(runner_worked_file))
+
+    with httmock.HTTMock(computer.request_handler):
+      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
+      self.assertInstanceDirectoryListEqual(['0'])
+      assertRunnerWorked()
+      six.assertCountEqual(self, os.listdir(partition.partition_path),
+                            ['.slapgrid', '.0_runner.log', 'buildout.cfg', 'etc',
+                             'runner_worked', 'software_release', 'worked',
+                             '.slapos-retention-lock-delay'])
+      runner_log_path = os.path.join(partition.partition_path, '.0_runner.log')
+      with open(runner_log_path) as f:
+        runner_log = f.read()
+      self.assertEqual(runner_log, 'Working\n')
+      self.assertEqual(partition.state, 'started')
+
+    computer.status_code = 503 # connection loss
+    os.unlink(runner_worked_file)
+
+    with httmock.HTTMock(computer.request_handler):
+      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_OFFLINE_SUCCESS)
+      self.assertInstanceDirectoryListEqual(['0'])
+      assertRunnerWorked()
+      with open(runner_log_path) as f:
+        runner_log = f.read()
+      self.assertEqual(runner_log, 'Working\n' * 2)
+      self.assertEqual(computer.sequence, [
+        '/getFullComputerInformation',
+        '/getComputerPartitionCertificate',
+        '/startedComputerPartition',
+        '/getComputerPartitionCertificate' # /getFullComputerInformation is cached
+      ])
 
 class TestSlapgridCPWithMasterWatchdog(MasterMixin, unittest.TestCase):
 
@@ -1401,7 +1466,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       six.assertCountEqual(self, os.listdir(partition),
-                            ['.slapgrid', '.timestamp', '.requested_state', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
+                            ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       timestamp_path = os.path.join(instance.partition_path, '.timestamp')
       self.setSlapgrid()
@@ -1422,7 +1487,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       six.assertCountEqual(self, os.listdir(partition),
-                            ['.slapgrid', '.timestamp', '.requested_state', 'buildout.cfg',
+                            ['.slapgrid', '.timestamp', 'buildout.cfg',
                              'software_release', 'worked', '.slapos-retention-lock-delay'])
       six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
 
@@ -1445,7 +1510,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       six.assertCountEqual(self, os.listdir(partition),
-                            ['.slapgrid', '.timestamp', '.requested_state', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
+                            ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       instance.timestamp = str(int(timestamp) - 1)
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
@@ -1463,7 +1528,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       six.assertCountEqual(self, os.listdir(partition),
-                            ['.slapgrid', '.timestamp', '.requested_state', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
+                            ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
       instance.timestamp = str(int(timestamp) + 1)
       self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
@@ -1491,7 +1556,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.assertInstanceDirectoryListEqual(['0'])
       partition = os.path.join(self.instance_root, '0')
       six.assertCountEqual(self, os.listdir(partition),
-                            ['.slapgrid', '.timestamp', '.requested_state', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
+                            ['.slapgrid', '.timestamp', 'buildout.cfg', 'software_release', 'worked', '.slapos-retention-lock-delay'])
       six.assertCountEqual(self, os.listdir(self.software_root),
                             [instance.software.software_hash])
       instance.timestamp = None
@@ -1523,7 +1588,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
       self.launchSlapgrid()
       partition = os.path.join(self.instance_root, '0')
       six.assertCountEqual(self, os.listdir(partition),
-                            ['.slapgrid', '.timestamp', '.requested_state', 'buildout.cfg',
+                            ['.slapgrid', '.timestamp', 'buildout.cfg',
                              'software_release', 'worked', '.slapos-retention-lock-delay'])
 
       time.sleep(2)
@@ -1533,7 +1598,7 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
 
       self.launchSlapgrid()
       six.assertCountEqual(self, os.listdir(partition),
-                            ['.slapgrid', '.timestamp', '.requested_state', 'buildout.cfg',
+                            ['.slapgrid', '.timestamp', 'buildout.cfg',
                              'software_release', 'worked', '.slapos-retention-lock-delay'])
 
   def test_one_partition_periodicity_from_file_does_not_disturb_others(self):
@@ -1710,43 +1775,6 @@ class TestSlapgridCPPartitionProcessing(MasterMixin, unittest.TestCase):
         self.launchSlapgrid()
         self.assertEqual(mock_method.call_count, 2)
 
-  def test_partition_requested_state_created(self):
-    computer = self.getTestComputerClass()(self.software_root, self.instance_root)
-    with httmock.HTTMock(computer.request_handler):
-      instance = computer.instance_list[0]
-      timestamp = str(int(time.time()))
-      instance.timestamp = timestamp
-
-      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_SUCCESS)
-      self.assertInstanceDirectoryListEqual(['0'])
-      partition = os.path.join(self.instance_root, '0')
-      six.assertCountEqual(self, os.listdir(partition),
-                            ['.slapgrid', '.timestamp', '.requested_state', 'buildout.cfg',
-                             'software_release', 'worked', '.slapos-retention-lock-delay'])
-      six.assertCountEqual(self, os.listdir(self.software_root), [instance.software.software_hash])
-      requested_state_path = os.path.join(instance.partition_path, '.requested_state')
-
-      with open(requested_state_path) as f:
-        self.assertEqual(f.read(), slapgrid.COMPUTER_PARTITION_STOPPED_STATE)
-      self.assertEqual(instance.sequence,
-                       ['/stoppedComputerPartition'])
-
-  def test_partition_requested_state_not_created_if_failed(self):
-    computer = self.getTestComputerClass()(self.software_root, self.instance_root)
-    with httmock.HTTMock(computer.request_handler):
-      instance = computer.instance_list[0]
-      timestamp = str(int(time.time()))
-      instance.timestamp = timestamp
-
-      instance.software.setBuildout("""#!/bin/sh
-exit 3""")
-      self.assertEqual(self.grid.processComputerPartitionList(), slapgrid.SLAPGRID_FAIL)
-      self.assertInstanceDirectoryListEqual(['0'])
-      self.assertEqual(instance.sequence,
-                       ['/softwareInstanceError'])
-      requested_state_path = os.path.join(instance.partition_path, '.requested_state')
-      self.assertFalse(os.path.exists(requested_state_path))
-
   def test_one_partition_buildout_fail_does_not_disturb_others(self):
     """
     1. We set up two instance one using a corrupted buildout
@@ -1915,6 +1943,50 @@ echo %s; echo %s; exit 42""" % (line1, line2))
       self.assertFalse(os.path.exists(promise_ran))
       self.assertFalse(instance.sequence)
 
+  def test_supervisor_partition_files_removed_on_stop(self):
+    computer = self.getTestComputerClass()(self.software_root, self.instance_root, 2, 1)
+    with httmock.HTTMock(computer.request_handler):
+      for i in range(2):
+        instance = computer.instance_list[i]
+        instance.requested_state = 'started'
+        run_dir = os.path.join(instance.partition_path, 'etc', 'run')
+        os.makedirs(run_dir)
+        with open(os.path.join(run_dir, 'runner' + str(i)), 'w'): pass
+      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
+
+      conf_path = os.path.join(self.instance_root, 'etc', 'supervisord.conf.d', '%s.conf')
+      conf0 = conf_path % 0
+      conf1 = conf_path % 1
+      for conf in (conf0, conf1):
+        self.assertTrue(os.path.exists(conf), conf + ' does not exist')
+
+      computer.instance_list[0].requested_state = 'stopped'
+      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
+
+      self.assertFalse(os.path.exists(conf0), conf0 + ' still exists')
+      self.assertTrue(os.path.exists(conf1), conf1 + ' does not exist')
+
+      computer.instance_list[0].requested_state = 'started'
+      computer.instance_list[1].requested_state = 'stopped'
+      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
+
+      self.assertTrue(os.path.exists(conf0), conf0 + ' does not exist')
+      self.assertFalse(os.path.exists(conf1), conf1 + ' still exists')
+
+      computer.instance_list[0].requested_state = 'stopped'
+      computer.instance_list[1].requested_state = 'stopped'
+      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
+
+      for conf in (conf0, conf1):
+        self.assertFalse(os.path.exists(conf), conf + ' still exists')
+
+      computer.instance_list[0].requested_state = 'started'
+      computer.instance_list[1].requested_state = 'started'
+      self.assertEqual(self.launchSlapgrid(), slapgrid.SLAPGRID_SUCCESS)
+
+      for conf in (conf0, conf1):
+        self.assertTrue(os.path.exists(conf), conf + ' does not exist')
+
 
 class TestSlapgridUsageReport(MasterMixin, unittest.TestCase):
   """
@@ -3030,13 +3102,17 @@ exit 0
       stat_info = os.stat(partition.partition_path)
       uid = stat_info.st_uid
       gid = stat_info.st_gid
-      supervisor_conf_file = os.path.join(self.instance_root,
+      partition_supervisor_conf_file = os.path.join(self.instance_root,
                                           'etc/supervisord.conf.d',
                                           '%s.conf' % partition.name)
-      self.assertTrue(os.path.exists(supervisor_conf_file))
+      prerm_supervisor_conf_file = os.path.join(self.instance_root,
+                                          'etc/supervisord.conf.d',
+                                          '%s-prerm.conf' % partition.name)
+      self.assertFalse(os.path.exists(partition_supervisor_conf_file))
+      self.assertTrue(os.path.exists(prerm_supervisor_conf_file))
       regex_user = r"user=(\d+)"
       regex_group = r"group=(\d+)"
-      with open(supervisor_conf_file) as f:
+      with open(prerm_supervisor_conf_file) as f:
         config = f.read()
         # search user uid in conf file
         result = re.search(regex_user, config, re.DOTALL)
@@ -3113,8 +3189,8 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
 
     self.computer = self.getTestComputerClass()(self.software_root, self.instance_root)
     self.partition = self.computer.instance_list[0]
-    self.instance_supervisord_config_path = os.path.join(
-      self.instance_root, 'etc/supervisord.conf.d/0.conf')
+    self.socat_supervisord_config_path = os.path.join(
+      self.instance_root, 'etc/supervisord.conf.d/0-socat.conf')
 
     self.port_redirect_path = os.path.join(self.partition.partition_path,
                                            slapmanager.portredir.Manager.port_redirect_filename)
@@ -3122,8 +3198,8 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
   def _mock_requests(self):
     return httmock.HTTMock(self.computer.request_handler)
 
-  def _read_instance_supervisord_config(self):
-    with open(self.instance_supervisord_config_path) as f:
+  def _read_socat_supervisord_config(self):
+    with open(self.socat_supervisord_config_path) as f:
       return f.read()
 
   def _setup_instance(self, config):
@@ -3151,9 +3227,9 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertIn('socat-tcp-{}'.format(1234), partition_supervisord_config)
-      self.assertIn('socat TCP4-LISTEN:1234,fork TCP4:127.0.0.1:4321', partition_supervisord_config)
+      socat_supervisord_config = self._read_socat_supervisord_config()
+      self.assertIn('socat-tcp-{}'.format(1234), socat_supervisord_config)
+      self.assertIn('socat TCP4-LISTEN:1234,fork TCP4:127.0.0.1:4321', socat_supervisord_config)
 
   def test_ipv6_port_redirection(self):
     with self._mock_requests():
@@ -3166,9 +3242,9 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertIn('socat-tcp-{}'.format(1234), partition_supervisord_config)
-      self.assertIn('socat TCP4-LISTEN:1234,fork TCP6:[::1]:4321', partition_supervisord_config)
+      socat_supervisord_config = self._read_socat_supervisord_config()
+      self.assertIn('socat-tcp-{}'.format(1234), socat_supervisord_config)
+      self.assertIn('socat TCP4-LISTEN:1234,fork TCP6:[::1]:4321', socat_supervisord_config)
 
   def test_udp_port_redirection(self):
     with self._mock_requests():
@@ -3182,9 +3258,9 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertIn('socat-udp-{}'.format(1234), partition_supervisord_config)
-      self.assertIn('socat UDP4-LISTEN:1234,fork UDP4:127.0.0.1:4321', partition_supervisord_config)
+      socat_supervisord_config = self._read_socat_supervisord_config()
+      self.assertIn('socat-udp-{}'.format(1234), socat_supervisord_config)
+      self.assertIn('socat UDP4-LISTEN:1234,fork UDP4:127.0.0.1:4321', socat_supervisord_config)
 
   def test_portredir_config_change(self):
     # We want the partition to just get updated, not recreated
@@ -3200,9 +3276,9 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertIn('socat-tcp-{}'.format(1234), partition_supervisord_config)
-      self.assertIn('socat TCP4-LISTEN:1234,fork TCP4:127.0.0.1:4321', partition_supervisord_config)
+      socat_supervisord_config = self._read_socat_supervisord_config()
+      self.assertIn('socat-tcp-{}'.format(1234), socat_supervisord_config)
+      self.assertIn('socat TCP4-LISTEN:1234,fork TCP4:127.0.0.1:4321', socat_supervisord_config)
 
       # Remove the port binding from config
       with open(self.port_redirect_path, 'w+') as f:
@@ -3219,9 +3295,7 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       self.assertEqual(self.partition.state, 'started')
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertNotIn('socat-tcp-{}'.format(1234), partition_supervisord_config)
-      self.assertNotIn('socat TCP4-LISTEN:1234,fork TCP4:127.0.0.1:4321', partition_supervisord_config)
+      self.assertFalse(os.path.exists(self.socat_supervisord_config_path))
 
   def test_port_redirection_config_bad_source_port(self):
     with self._mock_requests():
@@ -3234,9 +3308,7 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertNotIn('socat-tcp-bad', partition_supervisord_config)
-      self.assertNotIn('socat TCP4-LISTEN:bad,fork TCP4:127.0.0.1:4321', partition_supervisord_config)
+      self.assertFalse(os.path.exists(self.socat_supervisord_config_path))
 
   def test_port_redirection_config_bad_dest_port(self):
     with self._mock_requests():
@@ -3249,9 +3321,7 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertNotIn('socat-tcp-1234', partition_supervisord_config)
-      self.assertNotIn('socat TCP4-LISTEN:1234,fork TCP4:127.0.0.1:wolf', partition_supervisord_config)
+      self.assertFalse(os.path.exists(self.socat_supervisord_config_path))
 
   def test_port_redirection_config_bad_source_address(self):
     with self._mock_requests():
@@ -3265,9 +3335,7 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertNotIn('socat-tcp-1234', partition_supervisord_config)
-      self.assertNotIn('socat TCP4-LISTEN:1234,bind=bad,fork TCP4:127.0.0.1:4321', partition_supervisord_config)
+      self.assertFalse(os.path.exists(self.socat_supervisord_config_path))
 
   def test_port_redirection_config_bad_dest_address(self):
     with self._mock_requests():
@@ -3280,9 +3348,7 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertNotIn('socat-tcp-1234', partition_supervisord_config)
-      self.assertNotIn('socat TCP4-LISTEN:1234,fork TCP4:wolf:4321', partition_supervisord_config)
+      self.assertFalse(os.path.exists(self.socat_supervisord_config_path))
 
   def test_port_redirection_config_bad_redir_type(self):
     with self._mock_requests():
@@ -3296,9 +3362,7 @@ class TestSlapgridWithPortRedirection(MasterMixin, unittest.TestCase):
       ])
 
       # Check the socat command
-      partition_supervisord_config = self._read_instance_supervisord_config()
-      self.assertNotIn('socat-htcpcp-1234', partition_supervisord_config)
-      self.assertNotIn('socat HTCPCP4-LISTEN:1234,fork HTCPCP4:127.0.0.1:4321', partition_supervisord_config)
+      self.assertFalse(os.path.exists(self.socat_supervisord_config_path))
 
 
 class TestSlapgridWithDevPermLsblk(MasterMixin, unittest.TestCase):