Introducing basic process manager for runner

slapos/runner/__init__.py

@@ -8,6 +8,7 @@ import logging
 import logging.handlers
 from optparse import OptionParser, Option
 import os
+import slapos.runner.process
 import sys
 from slapos.runner.utils import runInstanceWithLock
 
@@ -144,5 +145,6 @@ def serve(config):
   if not os.path.exists(software_link):
     os.mkdir(software_link)
   runInstanceWithLock(app.config)
+  slapos.runner.process.setHandler()
   app.run(host=config.runner_host, port=int(config.runner_port),
       debug=config.debug, threaded=True)

slapos/runner/process.py

+import os
+import psutil
+import signal
+import subprocess
+import sys
+
+SLAPRUNNER_PROCESS_LIST = []
+
+
+class Popen(subprocess.Popen):
+  """
+  Extension of Popen to launch and kill process in a clean way
+  """
+  def __init__(self, *args, **kwargs):
+    """
+    Launch process and add object to process list for handler
+    """
+    self.name = kwargs.pop('name', None)
+    kwargs['stdin'] = subprocess.PIPE
+    kwargs['stderr'] = subprocess.STDOUT
+    kwargs.setdefault('stdout', subprocess.PIPE)
+    kwargs.setdefault('close_fds', True)
+    subprocess.Popen.__init__(self, *args, **kwargs)
+    SLAPRUNNER_PROCESS_LIST.append(self)
+    self.stdin.flush()
+    self.stdin.close()
+    self.stdin = None
+
+  def kill(self, sig=signal.SIGTERM, recursive=False):
+    """
+    Kill process and all its descendant if recursive
+    """
+    if self.poll() is None:
+      if recursive:
+        childs_pids = pidppid(self.pid)
+        for pid in childs_pids:
+          killNoFail(pid, sig)
+      killNoFail(self.pid, sig)
+      if self.poll() is not None:
+        SLAPRUNNER_PROCESS_LIST.remove(self)
+
+  def __del__(self):
+    """
+    Del function, try to kill process group
+    and process if its group does not exist
+    """
+    for pid in (-self.pid, self.pid):
+      try:
+        os.kill(-self.pid, 15)
+      except OSError:
+        pass
+    subprocess.Popen.__del__(self)
+
+
+def pidppid(pid, recursive=True):
+  """
+  Return a list of all children of pid
+  """
+  return [p.pid for p in psutil.Process(pid).get_children(recursive=recursive)]
+
+
+def killNoFail(pid, sig):
+  """
+  function to kill without failing. Return True if kill do not fail
+  """
+  try:
+    os.kill(pid, sig)
+    return True
+  except OSError:
+    return False
+
+
+def isRunning(name):
+  """
+  Return True if a process with this name is running
+  """
+  for process in SLAPRUNNER_PROCESS_LIST:
+    if process.name == name:
+      if process.poll() is None:
+        return True
+  return False
+
+
+def killRunningProcess(name, recursive=False):
+  """
+  Kill all process with name
+  """
+  for process in SLAPRUNNER_PROCESS_LIST:
+    if process.name == name:
+      process.kill(recursive=recursive)
+
+
+def handler(sig, frame):
+  """
+  Signal handler to kill all process
+  """
+  pid = os.getpid()
+  os.kill(-pid, sig)
+  sys.exit()
+
+
+def setHandler(sig_list=None):
+  if sig_list is None:
+    sig_list = [signal.SIGTERM]
+  for sig in sig_list:
+    signal.signal(sig, handler)

slapos/runner/static/js/scripts/process.js

@@ -83,10 +83,10 @@ function stopProcess() {
         sendStop = true;
 
         var urlfor = $SCRIPT_ROOT + "stopSlapgrid",
-            type = "slapgrid-sr.pid";
+            type = "slapgrid-sr";
 
         if ($("#instrun").text() === "Stop") {
-            type = "slapgrid-cp.pid";
+            type = "slapgrid-cp";
         }
         $.post(urlfor, {type: type}, function (data) {
             //if (data.result) {

slapos/runner/utils.py

@@ -7,10 +7,9 @@ import md5
 import logging
 import multiprocessing
 import re
-import signal
+from slapos.runner.process import Popen, isRunning, killRunningProcess
 import shutil
 import os
-import subprocess
 import time
 import urllib
 from xml.dom import minidom
@@ -20,22 +19,12 @@ from flask import jsonify
 
 import slapos.slap
 
+
 # Setup default flask (werkzeug) parser
 
 logger = logging.getLogger('werkzeug')
 
 
-class Popen(subprocess.Popen):
-  def __init__(self, *args, **kwargs):
-    kwargs['stdin'] = subprocess.PIPE
-    kwargs['stderr'] = subprocess.STDOUT
-    kwargs.setdefault('stdout', subprocess.PIPE)
-    kwargs.setdefault('close_fds', True)
-    subprocess.Popen.__init__(self, *args, **kwargs)
-    self.stdin.flush()
-    self.stdin.close()
-    self.stdin = None
-
 html_escape_table = {
   "&": "&",
   '"': """,
@@ -177,54 +166,6 @@ def updateProxy(config):
   computer.updateConfiguration(xml_marshaller.xml_marshaller.dumps(slap_config))
   return True
 
-def readPid(file):
-  """Read process pid from file `file`"""
-  if os.path.exists(file):
-    data = open(file).read().strip()
-    try:
-      return int(data)
-    except Exception:
-      return 0
-  return 0
-
-
-def writePid(file, pid):
-  """Save process pid into a file `file`"""
-  open(file, 'w').write(str(pid))
-
-
-def killRunningProcess(config, ptype):
-  """Kill process and all running children process and remove pidfile"""
-  process_pid = os.path.join(config['run_dir'], ptype)
-  pid = readPid(process_pid)
-  if pid:
-      recursifKill([pid])
-      os.remove(process_pid)
-  else:
-    return False
-
-def recursifKill(pids):
-  """Try to kill a list of proccess by the given pid list"""
-  if pids == []:
-    return
-  else:
-    for pid in pids:
-      ppids = pidppid(pid)
-      try:
-        os.kill(pid, signal.SIGKILL) #kill current process
-      except Exception:
-        pass
-      recursifKill(ppids) #kill all children of this process
-
-def pidppid(pid):
-  """get the list of the children pids of a process `pid`"""
-  proc = Popen('ps -o pid,ppid ax | grep "%d"' % pid, shell=True,
-               stdout=subprocess.PIPE)
-  ppid  = [x.split() for x in proc.communicate()[0].split("\n") if x]
-  return list(int(p) for p, pp in ppid if int(pp) == pid)
-
-
-
 def updateInstanceParameter(config, software_type=None):
   """
   Reconfigure Slapproxy to re-deploy current Software Instance with parameters.
@@ -238,27 +179,17 @@ def updateInstanceParameter(config, software_type=None):
 
 def startProxy(config):
   """Start Slapproxy server"""
-  proxy_pid = os.path.join(config['run_dir'], 'proxy.pid')
-  pid = readPid(proxy_pid)
-  running = False
-  if pid:
-    try:
-      os.kill(pid, 0)
-    except Exception:
-      pass
-    else:
-      running = True
-  if not running:
+  if not isRunning('slapproxy'):
     log = os.path.join(config['log_dir'], 'slapproxy.log')
-    proxy = Popen([config['slapproxy'], '--log_file', log,
-                  config['configuration_file_path']])
-    writePid(proxy_pid, proxy.pid)
+    Popen([config['slapproxy'], '--log_file', log,
+           config['configuration_file_path']],
+          name='slapproxy')
     time.sleep(4)
 
 
 def stopProxy(config):
   """Stop Slapproxy server"""
-  killRunningProcess(config,'proxy.pid')
+  pass
 
 
 def removeProxyDb(config):
@@ -267,22 +198,11 @@ def removeProxyDb(config):
   if os.path.exists(config['database_uri']):
     os.unlink(config['database_uri'])
 
-def isSoftwareRunning(config):
+def isSoftwareRunning(config=None):
   """
     Return True if slapgrid-sr is still running and false if slapgrid if not
   """
-  slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-sr.pid')
-  pid = readPid(slapgrid_pid)
-  if pid:
-    try:
-      os.kill(pid, 0)
-    except Exception:
-      running = False
-    else:
-      running = True
-  else:
-    running = False
-  return running
+  return isRunning('slapgrid-sr')
 
 
 def runSoftwareWithLock(config):
@@ -291,7 +211,7 @@ def runSoftwareWithLock(config):
     compilation is done
   """
   slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-sr.pid')
-  if not isSoftwareRunning(config):
+  if not isSoftwareRunning():
     if not os.path.exists(config['software_root']):
       os.mkdir(config['software_root'])
     stopProxy(config)
@@ -304,9 +224,10 @@ def runSoftwareWithLock(config):
     environment = os.environ.copy()
     environment['MAKEFLAGS'] = '-j%r' % multiprocessing.cpu_count()
     slapgrid = Popen([config['slapgrid_sr'], '-vc',
-                      '--pidfile',slapgrid_pid,
+                      '--pidfile', slapgrid_pid,
                       config['configuration_file_path'], '--now', '--develop'],
-                     stdout=logfile, env=environment)
+                     stdout=logfile, env=environment,
+                     name='slapgrid-sr')
     slapgrid.wait()
     #Saves the current compile software for re-use
     config_SR_folder(config)
@@ -362,22 +283,11 @@ def loadSoftwareRList(config):
       list.append(dict(md5=cfg[1], path=cfg[0], title=path))
   return list
 
-def isInstanceRunning(config):
+def isInstanceRunning(config=None):
   """
     Return True if slapgrid-cp is still running and false if slapgrid if not
   """
-  slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-cp.pid')
-  pid = readPid(slapgrid_pid)
-  if pid:
-    try:
-      os.kill(pid, 0)
-    except Exception:
-      running = False
-    else:
-      running = True
-  else:
-    running = False
-  return running
+  return isRunning('slapgrid-cp')
 
 
 def runInstanceWithLock(config):
@@ -386,16 +296,16 @@ def runInstanceWithLock(config):
     deployment is done.
   """
   slapgrid_pid = os.path.join(config['run_dir'], 'slapgrid-cp.pid')
-  if not isInstanceRunning(config):
+  if not isInstanceRunning():
     startProxy(config)
     logfile = open(config['instance_log'], 'w')
     if not (updateProxy(config) and requestInstance(config)):
       return False
     svcStopAll(config) #prevent lost control of process
     slapgrid = Popen([config['slapgrid_cp'], '-vc',
-                      '--pidfile',slapgrid_pid,
+                      '--pidfile', slapgrid_pid,
                       config['configuration_file_path'], '--now'],
-                     stdout=logfile)
+                     stdout=logfile, name='slapgrid-cp')
     slapgrid.wait()
     return True
   return False
@@ -587,10 +497,10 @@ def configNewSR(config, projectpath):
   """
   folder = realpath(config, projectpath)
   if folder:
-    if isInstanceRunning(config):
-      killRunningProcess(config, "slapgrid-cp.pid")
-    if isSoftwareRunning(config):
-      killRunningProcess(config, "slapgrid-sr.pid")
+    if isInstanceRunning():
+      killRunningProcess('slapgrid-cp')
+    if isSoftwareRunning():
+      killRunningProcess('slapgrid-sr')
     stopProxy(config)
     removeProxyDb(config)
     startProxy(config)
@@ -672,7 +582,7 @@ def removeSoftwareByName(config, md5, folderName):
     config: slaprunner configuration
     foldername: the link name given to the software release
     md5: the md5 filename given by slapgrid to SR folder"""
-  if isSoftwareRunning(config) or isInstanceRunning(config):
+  if isSoftwareRunning() or isInstanceRunning():
     raise Exception("Software installation or instantiation in progress, cannot remove")
   path = os.path.join(config['software_root'], md5)
   linkpath = os.path.join(config['software_link'], folderName)

slapos/runner/views.py

@@ -11,9 +11,10 @@ from flaskext.auth import Auth, AuthUser, login_required, logout
 from flask import (Flask, request, redirect, url_for, render_template,
                    g, flash, jsonify, session, abort, send_file)
 
+from slapos.runner.process import killRunningProcess
 from slapos.runner.utils import (checkSoftwareFolder, configNewSR, getFolder, getFolderContent, getProfilePath,
                                  getProjectList, getProjectTitle, getSession, getSlapStatus, getSvcStatus,
-                                 getSvcTailProcess, isInstanceRunning, isSoftwareRunning, isText, killRunningProcess,
+                                 getSvcTailProcess, isInstanceRunning, isSoftwareRunning, isText,
                                  loadSoftwareRList, md5sum, newSoftware, readFileFrom, readParameters, realpath,
                                  removeInstanceRoot, removeProxyDb, removeSoftwareByName, runInstanceWithLock,
                                  runSoftwareWithLock, saveSession, svcStartStopProcess, svcStopAll, tail,
@@ -427,7 +428,7 @@ def slapgridResult():
 
 @login_required()
 def stopSlapgrid():
-  result = killRunningProcess(app.config, request.form['type'])
+  result = killRunningProcess(request.form['type'])
   return jsonify(result=result)
 
 @login_required()