move slapos monitor script to slapos.toolbox

setup.py

@@ -45,6 +45,7 @@ setup(name=name,
         'GitPython', #needed for git manipulation into slaprunner
         'passlib',
         'netifaces',
+        'PyRSS2Gen',
       ] + additional_install_requires,
       extras_require = {
         'lampconfigure':  ["mysqlclient"], #needed for MySQL Database access
@@ -72,6 +73,12 @@ setup(name=name,
           'is-local-tcp-port-opened = slapos.promise.is_local_tcp_port_opened:main',
           'is-process-older-than-dependency-set = slapos.promise.is_process_older_than_dependency_set:main',
           'killpidfromfile = slapos.systool:killpidfromfile', # BBB
+          'monitor.bootstrap = slapos.monitor.monitor:main',
+          'monitor.collect = slapos.monitor.collect:main',
+          'monitor.runpromise = slapos.monitor.runpromise:main',
+          'monitor.genstatus = slapos.monitor.globalstate:main',
+          'monitor.genrss = slapos.monitor.status2rss:main',
+          'monitor.genconfig = slapos.monitor.monitor_gen_update_config:main',
           'runResiliencyUnitTestTestNode = slapos.resiliencytest:runUnitTest',
           'runResiliencyScalabilityTestNode = slapos.resiliencytest:runResiliencyTest',
           'runStandaloneResiliencyTest = slapos.resiliencytest:runStandaloneResiliencyTest',

slapos/monitor/globalstate.py

+#!/usr/bin/env python
+
+import sys
+import os
+import glob
+import json
+import ConfigParser
+import time
+from datetime import datetime
+
+def softConfigGet(config, *args, **kwargs):
+  try:
+    return config.get(*args, **kwargs)
+  except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+    return ""
+
+def generateStatisticsData(stat_file_path, content):
+  # csv document for statictics
+  if not os.path.exists(stat_file_path):
+    with open(stat_file_path, 'w') as fstat:
+      data_dict = {
+        "date": time.time(),
+        "data": ["Date, Success, Error, Warning"]
+      }
+      fstat.write(json.dumps(data_dict))
+
+  current_state = ''
+  if content.has_key('state'):
+    current_state = '%s, %s, %s, %s' % (
+      content['date'],
+      content['state']['success'],
+      content['state']['error'],
+      content['state']['warning'])
+
+  # append to file
+  if current_state:
+    with open (stat_file_path, mode="r+") as fstat:
+      fstat.seek(0,2)
+      position = fstat.tell() -2
+      fstat.seek(position)
+      fstat.write('%s}' % ',"{}"]'.format(current_state))
+
+def run(args_list):
+  monitor_file, instance_file = args_list
+
+  monitor_config = ConfigParser.ConfigParser()
+  monitor_config.read(monitor_file)
+
+  base_folder = monitor_config.get('monitor', 'private-folder')
+  status_folder = monitor_config.get('monitor', 'public-folder')
+  base_url = monitor_config.get('monitor', 'base-url')
+  related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
+  statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
+  parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
+
+  report_date = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
+
+  if not os.path.exists(statistic_folder):
+    try:
+      os.makedirs(statistic_folder)
+    except OSError, e:
+      if e.errno == os.errno.EEXIST and os.path.isdir(statistic_folder):
+        pass
+      else: raise
+
+  # search for all status files
+  file_list = filter(os.path.isfile,
+      glob.glob("%s/*.status.json" % status_folder)
+    )
+  error = warning = success = 0
+  status = 'OK'
+  promise_list = []
+  global_state_file = os.path.join(base_folder, 'monitor.global.json')
+  public_state_file = os.path.join(status_folder, 'monitor.global.json')
+  for file in file_list:
+    try:
+      with open(file, 'r') as temp_file:
+        tmp_json = json.loads(temp_file.read())
+    except ValueError:
+      # bad json file ?
+      continue
+    if tmp_json['status'] == 'ERROR':
+      error  += 1
+    elif tmp_json['status'] == 'OK':
+      success += 1
+    elif tmp_json['status'] == 'WARNING':
+      warning += 1
+    tmp_json['time'] = tmp_json['start-date'].split(' ')[1]
+    promise_list.append(tmp_json)
+
+  if error:
+    status = 'ERROR'
+  elif warning:
+    status = 'WARNING'
+
+  global_state_dict = dict(
+      status=status,
+      state={
+        'error': error,
+        'success': success,
+        'warning': warning,
+      },
+      type='global',
+      date=report_date,
+      _links={"rss_url": {"href": "%s/public/feed" % base_url},
+              "public_url": {"href": "%s/share/jio_public/" % base_url},
+              "private_url": {"href": "%s/share/jio_private/" % base_url}
+            },
+      data={'state': 'monitor_state.data',
+            'process_state': 'monitor_process_resource.status',
+            'process_resource': 'monitor_resource_process.data',
+            'memory_resource': 'monitor_resource_memory.data',
+            'io_resource': 'monitor_resource_io.data',
+            'monitor_process_state': 'monitor_resource.status'}
+    )
+
+  global_state_dict['_embedded'] = {'promises': promise_list}
+
+  if os.path.exists(instance_file):
+    config = ConfigParser.ConfigParser()
+    config.read(instance_file)
+    if 'instance' in config.sections():
+      instance_dict = {}
+      global_state_dict['title'] = config.get('instance', 'name')
+      global_state_dict['hosting-title'] = config.get('instance', 'root-name')
+      if not global_state_dict['title']:
+        global_state_dict['title'] = 'Instance Monitoring'
+      
+      instance_dict['computer'] = config.get('instance', 'computer')
+      instance_dict['ipv4'] = config.get('instance', 'ipv4')
+      instance_dict['ipv6'] = config.get('instance', 'ipv6')
+      instance_dict['software-release'] = config.get('instance', 'software-release')
+      instance_dict['software-type'] = config.get('instance', 'software-type')
+      instance_dict['partition'] = config.get('instance', 'partition')
+
+      global_state_dict['_embedded'].update({'instance' : instance_dict})
+
+  if related_monitor_list:
+    global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/jio_public" % url}
+                          for url in related_monitor_list]
+
+  if os.path.exists(parameter_file):
+    with open(parameter_file) as cfile:
+      global_state_dict['parameters'] = json.loads(cfile.read())
+
+  # Public information with the link to private folder
+  public_state_dict = dict(
+    status=status,
+    date=report_date,
+    _links={'monitor': {'href': '%s/share/jio_private/' % base_url}},
+    title=global_state_dict.get('title', '')
+  )
+  public_state_dict['hosting-title'] = global_state_dict.get('hosting-title', '')
+  public_state_dict['_links']['related_monitor'] = global_state_dict['_links'].get('related_monitor', [])
+
+  with open(global_state_file, 'w') as fglobal:
+    fglobal.write(json.dumps(global_state_dict))
+
+  with open(public_state_file, 'w') as fpglobal:
+    fpglobal.write(json.dumps(public_state_dict))
+
+  generateStatisticsData(
+    os.path.join(statistic_folder, 'monitor_state.data.json'),
+    global_state_dict)
+
+  return 0
+
+def main():
+  if len(sys.argv) < 3:
+    print("Usage: %s <monitor_conf_path> <instance_conf_path>" % sys.argv[0])
+    sys.exit(2)
+  sys.exit(run(sys.argv[1:]))

slapos/monitor/monitor_gen_update_config.py

+#!/usr/bin/env python
+
+import sys
+import os
+import re
+import json
+import argparse
+import subprocess
+from datetime import datetime
+import time
+
+
+def parseArguments():
+  """
+  Parse arguments for monitor instance.
+  """
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--config_folder',
+                      help='Path where json configuration/document will be read and write')
+  parser.add_argument('--htpasswd_bin',
+                      help='Path apache htpasswd binary. Needed to write htpasswd file.')
+  parser.add_argument('--output_cfg_file',
+                      help='Ouput parameters in cfg file.')
+
+  return parser.parse_args()
+
+def fileWrite(file_path, content):
+  if os.path.exists(file_path):
+    try:
+      with open(file_path, 'w') as wf:
+        wf.write(content)
+        return True
+    except OSError, e:
+      print "ERROR while writing changes to %s.\n %s" % (file_path, str(e))
+  return False
+
+def htpasswdWrite(htpasswd_bin,  parameter_dict, value):
+  if not os.path.exists(parameter_dict['file']):
+    return False
+  command = [htpasswd_bin, '-cb', parameter_dict['htpasswd'], parameter_dict['user'], value]
+  process = subprocess.Popen(
+    command,
+    stdin=None,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE
+  )
+  result = process.communicate()[0]
+  if process.returncode != 0:
+    print result
+    return False
+  with open(parameter_dict['file'], 'w') as pfile:
+    pfile.write(value)
+  return True
+
+def httpdCorsDomainWrite(httpd_cors_file, httpd_gracefull_bin, cors_domain):
+  cors_string = ""
+  cors_domain_list = cors_domain.split()
+  old_httpd_cors_file = os.path.join(
+    os.path.dirname(httpd_cors_file),
+    'prev_%s' % os.path.basename(httpd_cors_file)
+  )
+  if os.path.exists(old_httpd_cors_file) and os.path.isfile(old_httpd_cors_file):
+    try:
+      with open(old_httpd_cors_file, 'r') as cors_file:
+        if cors_file.read() == cors_domain:
+          if os.path.exists(httpd_cors_file) and (os.stat(httpd_cors_file).st_size > 0
+            or (cors_domain == "" and os.stat(httpd_cors_file).st_size == 0)):
+            # Skip if cors file is not empty
+            return True
+    except OSError, e:
+      print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
+  for domain in cors_domain_list:
+    if cors_string:
+      cors_string += '|'
+    cors_string += re.escape(domain)
+  try:
+    with open(httpd_cors_file, 'w') as file:
+      file.write('SetEnvIf Origin "^http(s)?://(.+\.)?(%s)$" origin_is=$0\n' % cors_string)
+      file.write('Header always set Access-Control-Allow-Origin %{origin_is}e env=origin_is')
+  except OSError, e:
+    print "ERROR while writing CORS changes to %s.\n %s" % (httpd_cors_file, str(e))
+    return False
+
+  # Save current cors domain list
+  try:
+    with open(old_httpd_cors_file, 'w') as cors_file:
+      cors_file.write(cors_domain)
+  except OSError, e:
+    print "Failed to open file at %s. \n%s" % (old_httpd_cors_file, str(e))
+    return False
+
+  # Restart httpd process
+  try:
+    subprocess.call(httpd_gracefull_bin)
+  except OSError, e:
+    print "Failed to execute command %s.\n %s" % (httpd_gracefull_bin, str(e))
+    return False
+
+def applyEditChage(parser):
+  parameter_tmp_file = os.path.join(parser.config_folder, 'config.tmp.json')
+  config_file = os.path.join(parser.config_folder, 'config.json')
+  parameter_config_file = os.path.join(parser.config_folder, 'config.parameters.json')
+  if not os.path.exists(parameter_tmp_file) or not os.path.isfile(parameter_tmp_file):
+    return {}
+  if not os.path.exists(config_file):
+    print "ERROR: Config file doesn't exist... Exiting"
+    return {}
+
+  new_parameter_list = []
+  parameter_list = []
+  description_dict = {}
+  result_dict = {}
+
+  try:
+    with open(parameter_tmp_file) as tmpfile:
+      new_parameter_list = json.loads(tmpfile.read())
+  except ValueError:
+    print "Error: Couldn't parse json file %s" % parameter_tmp_file
+
+  with open(parameter_config_file) as tmpfile:
+    description_dict = json.loads(tmpfile.read())
+
+  for i in range(0, len(new_parameter_list)):
+    key = new_parameter_list[i]['key']
+    if key != '':
+      description_entry = description_dict[key]
+      if description_entry['type'] == 'file':
+        result_dict[key] = fileWrite(description_entry['file'], new_parameter_list[i]['value'])
+      elif description_entry['type'] == 'htpasswd':
+        result_dict[key] = htpasswdWrite(parser.htpasswd_bin, description_entry, new_parameter_list[i]['value'])
+      elif description_entry['type'] == 'httpdcors':
+        result_dict[key] = httpdCorsDomainWrite(description_entry['cors_file'], description_entry['gracefull_bin'], new_parameter_list[i]['value'])
+
+  if (parser.output_cfg_file):
+    try:
+      with open(parser.output_cfg_file, 'w') as pfile:
+        pfile.write('[public]\n')
+        for parameter in new_parameter_list:
+          if parameter['key']:
+            pfile.write('%s = %s\n' % (parameter['key'], parameter['value']))
+    except OSError, e:
+      print "Error failed to create file %s" % parser.output_cfg_file
+      pass
+
+  return result_dict
+
+def main():
+  parser = parseArguments()
+  parameter_tmp_file = os.path.join(parser.config_folder, 'config.tmp.json')
+  config_file = os.path.join(parser.config_folder, 'config.json')
+
+  # Run 4 times with sleep
+  run_counter = 1
+  max_runn = 4
+  sleep_time = 15
+
+  while True:
+    result_dict = applyEditChage(parser)
+    if result_dict != {}:
+      status = True
+      for key in result_dict:
+        if not result_dict[key]:
+          status = False
+    
+      if status and os.path.exists(parameter_tmp_file):
+        try:
+          os.unlink(config_file)
+        except OSError, e:
+          print "ERROR cannot remove file: %s" % parameter_tmp_file
+        else:
+          os.rename(parameter_tmp_file, config_file)
+    if run_counter == max_runn:
+      break
+    else:
+      run_counter += 1
+      time.sleep(sleep_time)
+
+
+
+
+
+
+
+

slapos/monitor/runpromise.py

+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+import os
+import subprocess
+import json
+import psutil
+import time
+from shutil import copyfile
+import glob
+import argparse
+import traceback
+
+def parseArguments():
+  """
+  Parse arguments for monitor collector instance.
+  """
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--pid_path',
+                      help='Path where the pid of this process will be writen.')
+  parser.add_argument('--output',
+                      help='The Path of file where Json result of this promise will be saved.')
+  parser.add_argument('--promise_script',
+                      help='Promise script to execute.')
+  parser.add_argument('--promise_name',
+                      help='Title to give to this promise.')
+  parser.add_argument('--promise_type',
+                      default='status',
+                      help='Type of promise to execute. [status, report].')
+  parser.add_argument('--monitor_url',
+                      help='Monitor Instance website URL.')
+  parser.add_argument('--history_folder',
+                      help='Path where old result file will be placed before generate a new json result file.')
+  parser.add_argument('--instance_name',
+                      default='UNKNOWN Software Instance',
+                      help='Software Instance name.')
+  parser.add_argument('--hosting_name',
+                      default='UNKNOWN Hosting Subscription',
+                      help='Hosting Subscription name.')
+
+  return parser
+
+def runpromise(parser):
+
+  if os.path.exists(parser.pid_path):
+    with open(parser.pid_path, "r") as pidfile:
+      try:
+        pid = int(pidfile.read(6))
+      except ValueError:
+        pid = None
+      if pid and os.path.exists("/proc/" + str(pid)):
+        print("A process is already running with pid " + str(pid))
+        return 1
+  start_date = ""
+  with open(parser.pid_path, "w") as pidfile:
+    process = executeCommand(parser.promise_script)
+    ps_process = psutil.Process(process.pid)
+    start_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ps_process.create_time()))
+    pidfile.write(str(process.pid))
+
+  status_json = generateStatusJsonFromProcess(process, start_date=start_date)
+
+  status_json['_links'] = {"monitor": {"href": parser.monitor_url}}
+  status_json['title'] = parser.promise_name
+  status_json['instance'] = parser.instance_name
+  status_json['hosting_subscription'] = parser.hosting_name
+  status_json['type'] = parser.promise_type
+
+  # Save the lastest status change date (needed for rss)
+  status_json['change-time'] = ps_process.create_time()
+  if os.path.exists(parser.output):
+    with open(parser.output) as f:
+      try:
+        last_result = json.loads(f.read())
+        if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
+          status_json['change-time'] = last_result['change-time']
+      except ValueError:
+        pass
+
+  updateStatusHistoryFolder(
+    parser.promise_name,
+    parser.output,
+    parser.history_folder,
+    parser.promise_type
+  )
+  with open(parser.output, "w") as outputfile:
+    json.dump(status_json, outputfile)
+  os.remove(parser.pid_path)
+
+def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
+  history_path = os.path.join(history_folder)
+  if not os.path.exists(status_file):
+    return
+  if not os.path.exists(history_folder):
+    return
+  if not os.path.exists(history_path):
+    try:
+      os.makedirs(history_path)
+    except OSError, e:
+      if e.errno == os.errno.EEXIST and os.path.isdir(history_path):
+        pass
+      else: raise
+  with open(status_file, 'r') as sf:
+    try:
+      status_dict = json.loads(sf.read())
+    except ValueError:
+      traceback.print_exc()
+      return
+
+  if promise_type == 'status':
+    filename = '%s.history.json' % name
+    history_file = os.path.join(history_path, filename)
+    # Remove links from history (not needed)
+    status_dict.pop('_links', None)
+    if not os.path.exists(history_file):
+      with open(history_file, 'w') as f_history:
+        data_dict = {
+          "date": time.time(),
+          "data": [status_dict]
+        }
+        f_history.write(json.dumps(data_dict))
+    else:
+      # Remove useless informations
+      status_dict.pop('hosting_subscription', '')
+      status_dict.pop('title', '')
+      status_dict.pop('instance', '')
+      status_dict.pop('type', '')
+
+      with open (history_file, mode="r+") as f_history:
+        f_history.seek(0,2)
+        position = f_history.tell() -2
+        f_history.seek(position)
+        #f_history.write(',%s]}' % str(status_dict))
+        f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
+  elif promise_type == 'report':
+    # keep_item_amount = 3
+    filename = '%s.history.json' % (
+      name)
+
+    copyfile(status_file, os.path.join(history_path, filename))
+    """# Don't let history foler grow too much, keep xx files
+    file_list = filter(os.path.isfile,
+        glob.glob("%s/*.%s.history.json" % (history_path, promise_type))
+      )
+    file_count = len(file_list)
+    if file_count > keep_item_amount:
+      file_list.sort(key=lambda x: os.path.getmtime(x))
+      while file_count > keep_item_amount:
+        to_delete = file_list.pop(0)
+        try:
+          os.unlink(to_delete)
+          file_count -= 1
+        except OSError:
+          raise"""
+
+def generateStatusJsonFromProcess(process, start_date=None, title=None):
+  stdout, stderr = process.communicate()
+  status_json = {}
+  if process.returncode != 0:
+    status_json["status"] = "ERROR"
+  else:
+    status_json["status"] = "OK"
+  if stderr:
+    status_json["message"] = stderr
+  elif stdout:
+    status_json["message"] = stdout
+  if start_date:
+    status_json["start-date"] = start_date
+  if title:
+    status_json["title"] = title
+  return status_json
+
+
+def executeCommand(args):
+  return subprocess.Popen(
+    args,
+    #cwd=instance_path,
+    #env=None if sys.platform == 'cygwin' else {},
+    stdin=None,
+    stdout=subprocess.PIPE,
+    stderr=subprocess.PIPE
+  )
+
+def main():
+  arg_parser = parseArguments()
+  sys.exit(runpromise(parser.parse_args()))

slapos/monitor/status2rss.py

+import sys
+import os
+import json
+from datetime import datetime
+import base64
+import hashlib
+import PyRSS2Gen
+import argparse
+
+def parseArguments():
+  """
+  Parse arguments for monitor Rss Generator.
+  """
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--items_folder',
+                      help='Path where to get *.status.json files which contain result of promises.')
+  parser.add_argument('--output',
+                      help='The Path of file where feed file will be saved.')
+  parser.add_argument('--feed_url',
+                      help='Url of this feed file.')
+  parser.add_argument('--public_url',
+                      help='Monitor Instance public URL.')
+  parser.add_argument('--private_url',
+                      help='Monitor Instance private URL.')
+  parser.add_argument('--instance_name',
+                      default='UNKNOW Software Instance',
+                      help='Software Instance name.')
+  parser.add_argument('--hosting_name',
+                      default='',
+                      help='Hosting Subscription name.')
+
+  return parser.parse_args()
+
+def getKey(item):
+  return item.pubDate
+
+def main():
+  parser = parseArguments()
+
+  rss_item_list = []
+  report_date = datetime.utcnow()
+  for filename in os.listdir(parser.items_folder):
+    if filename.endswith(".status.json"):
+      filepath = os.path.join(parser.items_folder, filename)
+      result_dict = None
+      try:
+        result_dict = json.load(open(filepath, "r"))
+      except ValueError:
+        print "Failed to load json file: %s" % filepath
+        continue
+      description = result_dict.get('message', '')
+      event_time = datetime.fromtimestamp(result_dict['change-time'])
+      rss_item = PyRSS2Gen.RSSItem(
+        categories = [result_dict['status']],
+        source = PyRSS2Gen.Source(result_dict['title'], parser.public_url),
+        title = '[%s] %s' % (result_dict['status'], result_dict['title']),
+        comments = description,
+        description = "%s: %s\n%s" % (event_time, result_dict['status'], description),
+        link = parser.private_url,
+        pubDate = event_time,
+        guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (parser.hosting_name, result_dict['title'])))
+      )
+      rss_item_list.append(rss_item)
+
+
+  ### Build the rss feed
+  sorted(rss_item_list, key=getKey)
+  rss_feed = PyRSS2Gen.RSS2 (
+    title = parser.instance_name,
+    link = parser.feed_url,
+    description = parser.hosting_name,
+    lastBuildDate = report_date,
+    items = rss_item_list
+    )
+
+  with open(parser.output, 'w') as frss:
+    frss.write(rss_feed.to_xml())
+
+def main():
+  exit(main())

[Nicolas Wavrant]

@alain.takoudjou : is it me or does this script always run 4 times, even if first call to applyEditChage (instance.applyConfigChanges() nowadays) is succesfull ?

This behavior is the same currently : https://lab.nexedi.com/nexedi/slapos.toolbox/blob/1c8269b2b12414846dab010abbc289a99f1faf1d/slapos/monitor/monitor_config_write.py#L184

[Alain Takoudjou]

@Nicolas here, applyEditChage method will check if there is some configuration changes to apply. if yes it apply, else will just return. It checks 4 times because this script is run by cron (every minutes). So run 4 times help to apply a changes faster and not wait 1 minute in monitor interface to see if have a configuration applied.

[Nicolas Wavrant]

Ho right ! I already heard about it. Sorry for asking you again --'