slapos: Create supervisor configuration when running CLI.

Also move and simplify management of configuration files.

slapos/cli/configure_local/__init__.py

@@ -245,17 +245,6 @@ def do_configure(args, fetch_config_func, logger):
     configp = fetch_config_func(args)
     conf = FormatConfig(logger=logger)
     conf.mergeConfig(args, configp)
-    supervisord_socket_path = os.path.join(conf.instance_root,
-       'supervisord.socket')
-    supervisord_conf_path = os.path.join(conf.instance_root,
-       'etc', 'supervisord.conf')
-    conf_property_list = (
-      ('supervisord_socket', supervisord_socket_path),
-      ('supervisord_configuration_path', supervisord_conf_path),
-    )
-    for key, value in conf_property_list:
-        if not getattr(conf, key, None):
-            setattr(conf, key, value)
     slapgrid = create_slapgrid_object(conf.__dict__, logger)
     createPrivateDirectory(os.path.join(conf.slapos_buildout_directory, 'log'))
     _runFormat(conf.slapos_buildout_directory)
@@ -268,7 +257,6 @@ def do_configure(args, fetch_config_func, logger):
     slapos_client_cfg_path = '%s/.slapos/slapos-client.cfg' % home_folder_path
     if not os.path.exists(slapos_client_cfg_path):
         os.symlink(slapos_node_config_path, slapos_client_cfg_path)
-    launchSupervisord(socket=supervisord_socket_path,
-      configuration_file=supervisord_conf_path, logger=logger)
+    launchSupervisord(instance_root=conf.instance_root, logger=logger)
     _runFormat(conf.slapos_buildout_directory)
     return 0

slapos/cli/supervisorctl.py

@@ -28,13 +28,10 @@
 ##############################################################################
 
 import argparse
-import os
 
 from slapos.cli.command import check_root_user
 from slapos.cli.config import ConfigCommand
-from slapos.grid.svcbackend import launchSupervisord
-
-from slapos.util import string_to_boolean
+from slapos.grid.svcbackend import (launchSupervisord, _getSupervisordConfigurationFilePath)
 
 import supervisor.supervisorctl
 
@@ -66,11 +63,10 @@ class SupervisorctlCommand(ConfigCommand):
             check_root_user(self)
 
         instance_root = configp.get('slapos', 'instance_root')
-        configuration_file = os.path.join(instance_root, 'etc', 'supervisord.conf')
-        launchSupervisord(socket=os.path.join(instance_root, 'supervisord.socket'),
-                          configuration_file=configuration_file,
-                          logger=self.app.log)
-        supervisor.supervisorctl.main(args=['-c', configuration_file] + args.supervisor_args)
+        launchSupervisord(instance_root=instance_root, logger=self.app.log)
+        supervisor.supervisorctl.main(
+            args=['-c', _getSupervisordConfigurationFilePath(instance_root)] + args.supervisor_args
+        )
 
 
 class SupervisorctlAliasCommand(SupervisorctlCommand):

slapos/cli/supervisord.py

@@ -27,11 +27,8 @@
 #
 ##############################################################################
 
-import os
-
 from slapos.cli.config import ConfigCommand
-from slapos.grid.svcbackend import launchSupervisord
-
+from slapos.grid.svcbackend import (launchSupervisord, createSupervisordConfiguration)
 
 class SupervisordCommand(ConfigCommand):
     """
@@ -54,7 +51,8 @@ class SupervisordCommand(ConfigCommand):
           supervisord_additional_argument_list = ['--nodaemon']
         else:
           supervisord_additional_argument_list = []
-        launchSupervisord(socket=os.path.join(instance_root, 'supervisord.socket'),
-                          configuration_file=os.path.join(instance_root, 'etc', 'supervisord.conf'),
-                          logger=self.app.log,
-                          supervisord_additional_argument_list=supervisord_additional_argument_list)
+        createSupervisordConfiguration(instance_root)
+        launchSupervisord(
+            instance_root=instance_root, logger=self.app.log,
+            supervisord_additional_argument_list=supervisord_additional_argument_list
+        )

slapos/grid/slapgrid.py

@@ -37,7 +37,6 @@ import subprocess
 import sys
 import tempfile
 import time
-import stat
 import traceback
 import warnings
 import logging
@@ -53,9 +52,11 @@ from slapos.slap.slap import ServerError
 from slapos.util import mkdir_p, chownDirectory
 from slapos.grid.exception import BuildoutFailedError
 from slapos.grid.SlapObject import Software, Partition
-from slapos.grid.svcbackend import launchSupervisord
-from slapos.grid.utils import (md5digest, createPrivateDirectory, dropPrivileges,
-                               SlapPopen, updateFile)
+from slapos.grid.svcbackend import (launchSupervisord,
+                                    createSupervisordConfiguration,
+                                    _getSupervisordConfigurationDirectory,
+                                    _getSupervisordSocketPath)
+from slapos.grid.utils import (md5digest, dropPrivileges, SlapPopen)
 from slapos.human import human2bytes
 import slapos.slap
 
@@ -71,12 +72,12 @@ SLAPGRID_FAIL = 1
 SLAPGRID_PROMISE_FAIL = 2
 PROMISE_TIMEOUT = 3
 
-# XXX hardcoded watchdog_path
-WATCHDOG_PATH = '/opt/slapos/bin/slapos-watchdog'
-
 COMPUTER_PARTITION_TIMESTAMP_FILENAME = '.timestamp'
 COMPUTER_PARTITION_LATEST_BANG_TIMESTAMP_FILENAME = '.slapos_latest_bang_timestamp'
 
+# XXX hardcoded watchdog_path
+WATCHDOG_PATH = '/opt/slapos/bin/slapos-watchdog'
+
 
 class _formatXMLError(Exception):
   pass
@@ -149,15 +150,6 @@ def merged_options(args, configp):
   if options.get('all'):
     options['develop'] = True
 
-  # Supervisord configuration location
-  if not options.get('supervisord_configuration_path'):
-    options['supervisord_configuration_path'] = \
-      os.path.join(options['instance_root'], 'etc', 'supervisord.conf')
-  # Supervisord socket
-  if not options.get('supervisord_socket'):
-    options['supervisord_socket'] = \
-      os.path.join(options['instance_root'], 'supervisord.socket')
-
   # Parse cache / binary cache options
   # Backward compatibility about "binary-cache-url-blacklist" deprecated option
   if (options.get("binary-cache-url-blacklist") and not
@@ -209,8 +201,6 @@ def create_slapgrid_object(options, logger):
                   instance_root=op['instance_root'],
                   master_url=op['master_url'],
                   computer_id=op['computer_id'],
-                  supervisord_socket=op['supervisord_socket'],
-                  supervisord_configuration_path=op['supervisord_configuration_path'],
                   buildout=op.get('buildout'),
                   logger=logger,
                   maximum_periodicity = op.get('maximum_periodicity', 86400),
@@ -267,8 +257,6 @@ class Slapgrid(object):
                instance_root,
                master_url,
                computer_id,
-               supervisord_socket,
-               supervisord_configuration_path,
                buildout,
                logger,
                maximum_periodicity=86400,
@@ -303,8 +291,7 @@ class Slapgrid(object):
     self.instance_root = os.path.abspath(instance_root)
     self.master_url = master_url
     self.computer_id = computer_id
-    self.supervisord_socket = supervisord_socket
-    self.supervisord_configuration_path = supervisord_configuration_path
+    self.supervisord_socket = _getSupervisordSocketPath(instance_root)
     self.key_file = key_file
     self.cert_file = cert_file
     self.master_ca_file = master_ca_file
@@ -332,8 +319,6 @@ class Slapgrid(object):
         cert_file=self.cert_file, master_ca_file=self.master_ca_file)
     self.computer = self.slap.registerComputer(self.computer_id)
     # Defines all needed paths
-    self.supervisord_configuration_directory = \
-        os.path.join(self.instance_root, 'etc', 'supervisord.conf.d')
     self.buildout = buildout
     self.promise_timeout = promise_timeout
     self.develop = develop
@@ -350,7 +335,7 @@ class Slapgrid(object):
     self.software_min_free_space = software_min_free_space
     self.instance_min_free_space = instance_min_free_space
 
-  def getWatchdogLine(self):
+  def _getWatchdogLine(self):
     invocation_list = [WATCHDOG_PATH]
     invocation_list.append("--master-url '%s' " % self.master_url)
     if self.certificate_repository_path:
@@ -367,42 +352,11 @@ class Slapgrid(object):
     # Checks for software_root and instance_root existence
     if not os.path.isdir(self.software_root):
       raise OSError('%s does not exist.' % self.software_root)
-    if not os.path.isdir(self.instance_root):
-      raise OSError('%s does not exist.' % self.instance_root)
-    # Creates everything needed
-
-    # Create directory accessible for the instances.
-    var_directory = os.path.join(self.instance_root, 'var')
-    if not os.path.isdir(var_directory):
-      os.mkdir(var_directory)
-
-    os.chmod(var_directory, stat.S_IRWXU | stat.S_IROTH | stat.S_IXOTH | \
-                            stat.S_IRGRP | stat.S_IXGRP )
-
-    mkdir_p(os.path.join(self.instance_root, 'var'), 0o755)
-
-    # Creates instance_root structure
-    createPrivateDirectory(os.path.join(self.instance_root, 'var', 'log'))
-    createPrivateDirectory(os.path.join(self.instance_root, 'var', 'run'))
-
-    createPrivateDirectory(os.path.join(self.instance_root, 'etc'))
-    createPrivateDirectory(self.supervisord_configuration_directory)
-
-    # Creates supervisord configuration
-    updateFile(self.supervisord_configuration_path,
-      pkg_resources.resource_stream(__name__,
-        'templates/supervisord.conf.in').read() % {
-            'supervisord_configuration_directory': self.supervisord_configuration_directory,
-            'supervisord_socket': os.path.abspath(self.supervisord_socket),
-            'supervisord_loglevel': 'info',
-            'supervisord_logfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'log', 'supervisord.log')),
-            'supervisord_logfile_maxbytes': '50MB',
-            'supervisord_nodaemon': 'false',
-            'supervisord_pidfile': os.path.abspath(os.path.join(self.instance_root, 'var', 'run', 'supervisord.pid')),
-            'supervisord_logfile_backups': '10',
-            'watchdog_command': self.getWatchdogLine(),
-        }
-    )
+
+    createSupervisordConfiguration(self.instance_root, self._getWatchdogLine())
+
+  def _launchSupervisord(self):
+    launchSupervisord(instance_root=self.instance_root, logger=self.logger)
 
   def getComputerPartitionList(self):
     try:
@@ -501,11 +455,6 @@ class Slapgrid(object):
       return SLAPGRID_FAIL
     return SLAPGRID_SUCCESS
 
-  def _launchSupervisord(self):
-    launchSupervisord(self.supervisord_socket,
-                      self.supervisord_configuration_path,
-                      logger=self.logger)
-
   def _checkPromises(self, computer_partition):
     self.logger.info("Checking promises...")
     instance_path = os.path.join(self.instance_root, computer_partition.getId())
@@ -678,7 +627,7 @@ class Slapgrid(object):
         software_path=software_path,
         instance_path=instance_path,
         supervisord_partition_configuration_path=os.path.join(
-          self.supervisord_configuration_directory, '%s.conf' %
+          _getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' %
           computer_partition_id),
         supervisord_socket=self.supervisord_socket,
         computer_partition=computer_partition,
@@ -1136,7 +1085,7 @@ class Slapgrid(object):
             instance_path=os.path.join(self.instance_root,
                 computer_partition.getId()),
             supervisord_partition_configuration_path=os.path.join(
-              self.supervisord_configuration_directory, '%s.conf' %
+              _getSupervisordConfigurationDirectory(self.instance_root), '%s.conf' %
               computer_partition_id),
             supervisord_socket=self.supervisord_socket,
             computer_partition=computer_partition,

slapos/grid/svcbackend.py

@@ -29,14 +29,17 @@
 ##############################################################################
 
 import os
+import pkg_resources
+import socket as socketlib
+import subprocess
+import stat
 import sys
 import time
 import xmlrpclib
-import socket as socketlib
-import subprocess
+
+from slapos.grid.utils import (createPrivateDirectory, SlapPopen, updateFile)
 
 from supervisor import xmlrpc
-from slapos.grid.utils import SlapPopen
 
 
 def getSupervisorRPC(socket):
@@ -47,7 +50,65 @@ def getSupervisorRPC(socket):
   return getattr(server_proxy, 'supervisor')
 
 
-def launchSupervisord(socket, configuration_file, logger, supervisord_additional_argument_list=None):
+def _getSupervisordSocketPath(instance_root):
+  return os.path.join(instance_root, 'supervisord.socket')
+
+def _getSupervisordConfigurationFilePath(instance_root):
+  return os.path.join(instance_root, 'etc', 'supervisord.conf')
+
+def _getSupervisordConfigurationDirectory(instance_root):
+  return os.path.join(instance_root, 'etc', 'supervisord.conf.d')
+
+def createSupervisordConfiguration(instance_root, watchdog_command='sleep 10'):
+  """
+  Create supervisord related files and directories.
+  """
+  if not os.path.isdir(instance_root):
+    raise OSError('%s does not exist.' % instance_root)
+
+  supervisord_configuration_file_path = _getSupervisordConfigurationFilePath(instance_root)
+  supervisord_configuration_directory = _getSupervisordConfigurationDirectory(instance_root)
+  supervisord_socket = _getSupervisordSocketPath(instance_root)
+
+  # Create directory accessible for the instances.
+  var_directory = os.path.join(instance_root, 'var')
+  if not os.path.isdir(var_directory):
+    os.mkdir(var_directory)
+  os.chmod(var_directory, stat.S_IRWXU | stat.S_IROTH | stat.S_IXOTH | \
+                          stat.S_IRGRP | stat.S_IXGRP )
+  etc_directory = os.path.join(instance_root, 'etc')
+  if not os.path.isdir(etc_directory):
+    os.mkdir(etc_directory)
+
+  # Creates instance_root structure
+  createPrivateDirectory(os.path.join(instance_root, 'var', 'log'))
+  createPrivateDirectory(os.path.join(instance_root, 'var', 'run'))
+
+  createPrivateDirectory(os.path.join(instance_root, 'etc'))
+  createPrivateDirectory(supervisord_configuration_directory)
+
+  # Creates supervisord configuration
+  updateFile(supervisord_configuration_file_path,
+    pkg_resources.resource_stream(__name__,
+      'templates/supervisord.conf.in').read() % {
+          'supervisord_configuration_directory': supervisord_configuration_directory,
+          'supervisord_socket': os.path.abspath(supervisord_socket),
+          'supervisord_loglevel': 'info',
+          'supervisord_logfile': os.path.abspath(
+              os.path.join(instance_root, 'var', 'log', 'supervisord.log')),
+          'supervisord_logfile_maxbytes': '50MB',
+          'supervisord_nodaemon': 'false',
+          'supervisord_pidfile': os.path.abspath(
+              os.path.join(instance_root, 'var', 'run', 'supervisord.pid')),
+          'supervisord_logfile_backups': '10',
+          'watchdog_command': watchdog_command,
+      }
+  )
+
+def launchSupervisord(instance_root, logger,
+                      supervisord_additional_argument_list=None):
+  configuration_file = _getSupervisordConfigurationFilePath(instance_root)
+  socket = _getSupervisordSocketPath(instance_root)
   if os.path.exists(socket):
     trynum = 1
     while trynum < 6:
@@ -66,6 +127,18 @@ def launchSupervisord(socket, configuration_file, logger, supervisord_additional
       else:
         if status['statename'] == 'RUNNING' and status['statecode'] == 1:
           logger.debug('Supervisord already running.')
+
+          # Update watchdog
+          supervisor = getSupervisorRPC(socket)
+          try:
+            # XXX workaround for https://github.com/Supervisor/supervisor/issues/339
+            # In theory, only reloadConfig is needed.
+            supervisor.stopProcess('watchdog')
+            supervisor.removeProcessGroup('watchdog')
+          except:
+            pass
+          supervisor.reloadConfig()
+          supervisor.addProcessGroup('watchdog')
           return
         elif status['statename'] == 'SHUTDOWN_STATE' and status['statecode'] == 6:
           logger.info('Supervisor in shutdown procedure, will check again later.')
@@ -123,3 +196,4 @@ def launchSupervisord(socket, configuration_file, logger, supervisord_additional
     logger.warning('Issue while checking supervisord.')
   finally:
     socketlib.setdefaulttimeout(default_timeout)
+

slapos/grid/templates/supervisord.conf.in

@@ -22,3 +22,4 @@ chmod=0700
 [eventlistener:watchdog]
 command=%(watchdog_command)s
 events=PROCESS_STATE_EXITED, PROCESS_STATE_FATAL
+autorestart=true

slapos/tests/configure_local.py

@@ -38,8 +38,7 @@ from ConfigParser import ConfigParser
 
 # Disable any command to launch slapformat and supervisor
 slapos.cli.configure_local._runFormat = lambda x: "Do nothing"
-slapos.cli.configure_local.launchSupervisord = lambda socket, \
-  configuration_file, logger: "Do nothing"
+slapos.cli.configure_local.launchSupervisord = lambda instance_root, logger: "Do nothing"
 
 class TestConfigureLocal(unittest.TestCase):