monitor2 stack: first version for renderjs web interface

stack/monitor2/buildout.cfg

@@ -33,6 +33,12 @@ on-update = true
 <= monitor-download-base
 url = ${:_profile_base_location_}/templates/${:filename}
 
+[monitor-template-script]
+<= monitor-download-base
+url = ${:_profile_base_location_}/scripts/${:filename}
+destination = ${buildout:parts-directory}/monitor-scripts
+on-update = true
+
 [eggs]
 recipe = zc.recipe.egg
 eggs +=
@@ -40,9 +46,10 @@ eggs +=
   cns.recipe.symlink
 
 [extra-eggs]
-recipe = zc.recipe.egg
+<= eggs
 interpreter = pythonwitheggs
-eggs =
+eggs +=
+  psutil
   PyRSS2Gen
   Jinja2
 
@@ -85,7 +92,7 @@ md5sum = 04b664dfb47bfd3d01502768311aa239
 # Monitor templates files
 [monitor-httpd-conf]
 <= monitor-template-base
-md5sum = 8a1aa7cba281877d6cf63cb8ade64b5e
+md5sum = 68a7e4d3304fc78d173f30ad99534762
 filename = monitor-httpd.conf.in
 
 [monitor-service-conf-template]
@@ -102,6 +109,11 @@ md5sum = 8cde04bfd0c0e9bd56744b988275cfd8
 <= monitor-template-base
 filename = monitor.conf.in
 md5sum = 2db5c08c7e8658981b4b1e3f27fd5967
+
+[monitor-instance-info]
+<= monitor-template-base
+filename = instance-info.conf.in
+md5sum = d506e335e366c1d873a3490333842398
 # End templates files
 
 
@@ -110,12 +122,15 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = bf0578bb6863fea73c1c40bc009c7654
+md5sum = 8763a92c2c22103a9f6c9052ba322fc5
 context =
     key apache_location apache:location
     key gzip_location gzip:location
     raw monitor_bin ${monitor2-bin:location}/${monitor2-bin:filename}
+    raw monitor_collect ${monitor-collect:location}/${monitor-collect:filename}
     raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
+    raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
+    raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename}
     raw monitor_password_promise_template ${monitor-password-promise:location}/${monitor-password-promise:filename}
     raw monitor_password_cgi_template ${monitor-password-py-cgi:location}/${monitor-password-py-cgi:filename}
     raw monitor_password_promise_interface_template ${monitor-password-promise-interface:location}/${monitor-password-promise-interface:filename}
@@ -135,35 +150,44 @@ context =
     raw monitor_service_conf_template ${monitor-service-conf-template:location}/${monitor-service-conf-template:filename}
     raw openssl_executable_location ${openssl:location}/bin/openssl
     raw python_executable ${buildout:executable}
-    raw promise_executor_py ${run-promise-py:location}/${run-promise-py:filename}
+    raw python_with_eggs ${buildout:directory}/bin/${extra-eggs:interpreter}
+    raw promise_executor_py ${run-promise-py:rendered}
     raw template_wrapper ${template-wrapper:output}
     raw status2rss_executable_path ${status2rss-executable:location}/${status2rss-executable:filename}
 
 [monitor2-bin]
-<= monitor-download-base
+<= monitor-template-script
 filename = monitor.py
-md5sum = 7e1f2210a87b2212d11524e06985dd49
-url = ${:_profile_base_location_}/scripts/${:filename}
+md5sum = 3d2035d7c02b9d37c5c680e142dcb845
 
 [run-promise-py]
-<= monitor-download-base
-filename = run-promise.py
-md5sum = 8a46adcbc126ec9589d1810ba291d048
+recipe = slapos.recipe.template:jinja2
+template = ${:_profile_base_location_}/scripts/run-promise.py
+rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
+md5sum = 4f99c09125c201ee988e71662b8c9bf9
 mode = 0755
-url = ${:_profile_base_location_}/scripts/${:filename}
+context =
+  raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
 
 [monitor-password-promise]
-<= monitor-download-base
+<= monitor-template-script
 filename = monitor-password-promise.py
 md5sum = f7e937d6619eb674f39f34718928d91d
-url = ${:_profile_base_location_}/scripts/${:filename}
 
 [status2rss-executable]
-<= monitor-download-base
+<= monitor-template-script
 filename = status2rss.py
-md5sum = 65315ded80cd72f54f6e12d06ce813c4
-url = ${:_profile_base_location_}/scripts/${:filename}
+md5sum = 40dcda69f1ccb8639d7ef0ad65bceb5e
+
+[monitor-globalstate]
+<= monitor-template-script
+filename = globalstate.py
+md5sum = 906776302bfe511b21a7eee3c9bcb6a5
 
+[monitor-collect]
+<= monitor-template-script
+filename = collect.py
+md5sum = 8f4f36039ac8b30e59c11dea0bfeb65e
 
 
 [make-rss-script]

stack/monitor2/instance-monitor.cfg.jinja2.in

+[slap-configuration]
+recipe = slapos.cookbook:slapconfiguration.serialised
+computer = ${slap-connection:computer-id}
+partition = ${slap-connection:partition-id}
+url = ${slap-connection:server-url}
+key = ${slap-connection:key-file}
+cert = ${slap-connection:cert-file}
+
 [cron]
 recipe = slapos.cookbook:cron
 cron-entries = ${logrotate-directory:cron-entries}
@@ -58,6 +66,7 @@ run = ${directory:monitor}/run
 #run = ${directory:scripts}
 pids = ${directory:run}/monitor
 cgi-bin = ${directory:monitor}/cgi-bin
+webdav = ${directory:monitor}/webdav
 public = ${directory:monitor}/public
 private = ${directory:monitor}/private
 services = ${directory:services}
@@ -117,10 +126,13 @@ wrapper = ${directory:services}/monitor-httpd
 title = ${monitor-instance-parameter:monitor-title}
 public-folder = ${monitor-directory:public}
 private-folder = ${monitor-directory:private}
+webdav-folder = ${monitor-directory:webdav}
 web-folder = ${monitor-directory:web-dir}
-monitor-hal-json = ${monitor-directory:web-dir}/monitor.haljson
+base-url = ${monitor-httpd-conf-parameter:url}
+monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
 service-pid-folder = ${monitor-directory:pids}
 crond-folder = ${logrotate-directory:cron-entries}
+logrotate-folder = ${logrotate:logrotate-entries}
 wraper-folder = ${monitor-directory:promise-wrapper}
 promise-runner = {{ promise_executor_py }}
 promise-folder-list = 
@@ -128,11 +140,15 @@ promise-folder-list =
   ${directory:monitor-promise}
 
 public-path-list = 
-  ${directory:log}
 private-path-list = 
-
+    ${directory:log}
+# 
 monitor-url-list = 
 
+collector-db = ${monitor-instance-parameter:collector-db}
+collect-script = {{ monitor_collect }}
+python = {{ python_with_eggs }}
+
 [monitor-conf]
 recipe = slapos.recipe.template:jinja2
 template = {{ monitor_conf_template }}
@@ -140,10 +156,26 @@ rendered = ${directory:etc}/${:filename}
 filename = monitor.conf
 context = section parameter_dict monitor-conf-parameters
 
+[instance-info-parameters]
+name = ${monitor-instance-parameter:monitor-title}
+computer-id = ${slap-connection:computer-id}
+ipv4 = ${slap-configuration:ipv4-random}
+ipv6 = ${slap-configuration:ipv6-random}
+software-release = ${slap-connection:software-release-url}
+software-type = ${slap-configuration:slap-software-type}
+partition-id = ${slap-connection:partition-id}
+
+[monitor-instance-info]
+recipe = slapos.recipe.template:jinja2
+template = {{ monitor_instance_info }}
+rendered = ${directory:etc}/${:filename}
+filename = instance-info.conf
+context = section instance_dict instance-info-parameters
+
 [python-symlink]
 recipe = plone.recipe.command
 target = ${directory:bin}
-command = ln -sf {{ python_executable }} ${:target}/python
+command = ln -sf {{ python_with_eggs }} ${:target}/python
 update-command = ${:command}
 
 [start-monitor]
@@ -171,7 +203,8 @@ error-log = ${monitor-directory:log}/httpd-error.log
 cert-file = ${ca-directory:certs}/httpd.crt
 key-file = ${ca-directory:certs}/httpd.key
 htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path}
-url = https://[${monitor-instance-parameter:monitor-httpd-ipv6}]:${:port}/
+url = https://[${monitor-instance-parameter:monitor-httpd-ipv6}]:${:port}
+httpd-cors-config-file = ${monitor-httpd-cors:location}
 
 [monitor-httpd-conf]
 recipe = slapos.recipe.template:jinja2
@@ -182,6 +215,13 @@ context =
   section directory monitor-directory
   section parameter_dict monitor-httpd-conf-parameter
 
+[monitor-httpd-cors]
+recipe = plone.recipe.command
+command = if [ ! -f ${:location} ]; then echo 'Header set Access-Control-Allow-Origin "*"' > ${:location}; fi
+location = ${directory:etc}/${:filename}
+filename = httpd-cors.cfg
+stop-on-error = true
+
 [httpd-wrapper]
 recipe = slapos.cookbook:wrapper
 command-line = {{ apache_location }}/bin/httpd -f ${monitor-httpd-conf:rendered} -DFOREGROUND
@@ -202,7 +242,7 @@ command = kill -USR1 $(cat ${monitor-httpd-conf-parameter:pid-file})
 
 [monitor-status2rss-wrapper]
 recipe = slapos.cookbook:wrapper
-command-line = {{ python_executable }} {{ status2rss_executable_path }} '${monitor-instance-parameter:monitor-title}' '${monitor-httpd-conf-parameter:url}' ${monitor-directory:public} ${monitor-directory:monitor-status2rss-var}/previous_status ${monitor-directory:web-dir}/feed
+command-line = {{ python_with_eggs }} {{ status2rss_executable_path }} '${monitor-instance-parameter:monitor-title}' '${monitor-httpd-conf-parameter:url}' '${monitor-httpd-conf-parameter:url}/public' ${monitor-directory:public} ${monitor-directory:public}/feed
 wrapper-path = ${directory:bin}/monitor-status2rss.py
 
 [monitor-status2rss-cron-entry]
@@ -212,6 +252,18 @@ name = monitor-status2rss
 frequency = * * * * *
 command = ${monitor-status2rss-wrapper:wrapper-path}
 
+[monitor-globalstate-wrapper]
+recipe = slapos.cookbook:wrapper
+command-line = {{ python_with_eggs }} {{ monitor_globalstate }} '${monitor-conf:rendered}' '${monitor-instance-info:rendered}'
+wrapper-path = ${directory:bin}/monitor-globalstate
+
+[monitor-globalstate-cron-entry]
+recipe = slapos.cookbook:cron.d
+cron-entries = ${cron:cron-entries}
+name = monitor-globalstate
+frequency = * * * * *
+command = ${monitor-globalstate-wrapper:wrapper-path}
+
 [monitor-web-directory]
 recipe = plone.recipe.command
 command = cp -f {{ monitor_web_directory }}/* ${monitor-directory:web-dir}
@@ -252,8 +304,8 @@ context = section parameter_dict monitor-httpd-promise-conf-parameter
 
 [monitor-httpd-promise-conf-parameter]
 title = Monitor httpd listening
-# frequency minute hour day mounth weekday
-frequency = * * * * *
+# frequency 5 minute hour day mounth weekday
+frequency = */5 * * * *
 public-path-list = ${monitor-httpd-conf-parameter:access-log} ${monitor-httpd-conf-parameter:error-log}
 #private-path-list =
 
@@ -301,15 +353,23 @@ context =
 [publish]
 recipe = slapos.cookbook:publish
 monitor-url = ${monitor-httpd-conf-parameter:url}
+monitor-config-url = ${monitor-httpd-conf-parameter:url}/share/jio_public
 
 [monitor-instance-parameter]
 monitor-title = Monitoring interface
+# XXX Hard coded parameter
+collector-db = /srv/slapgrid/var/data-log/collector.db
+
+[monitor-base]
+# create dependencies between required monitor parts
+recipe = plone.recipe.command
+command = true
+update-command = 
+monitor-globalstate = ${monitor-globalstate-cron-entry:name}
 
 [buildout]
 parts =
-  monitor-web-directory
-  monitor-web-monitor-logout-cgi
-  monitor-web-monitor-promise-runner-cgi
+  monitor-base
   cron-entry-logrotate
   certificate-authority
   monitor-conf
@@ -317,9 +377,5 @@ parts =
   ca-httpd
   monitor-httpd-promise
   monitor-httpd-promise-conf
-  monitor-password-promise
-  monitor-password-promise-conf
-  monitor-password-cgi
-  monitor-password-promise-interface
   monitor-status2rss-cron-entry
   publish

stack/monitor2/scripts/collect.py

+# -*- coding: utf-8 -*-
+##############################################################################
+#
+# Copyright (c) 2010-2014 Vifib SARL and Contributors.
+# All Rights Reserved.
+#
+# WARNING: This program as such is intended to be used by professional
+# programmers who take the whole responsibility of assessing all potential
+# consequences resulting from its eventual inadequacies and bugs
+# End users who are looking for a ready-to-use solution with commercial
+# guarantees and support are strongly adviced to contract a Free Software
+# Service Company
+#
+# This program is Free Software; you can redistribute it and/or
+# modify it under the terms of the GNU Lesser General Public License
+# as published by the Free Software Foundation; either version 2.1
+# of the License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU Lesser General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA  02111-1307, USA.
+#
+##############################################################################
+
+import sqlite3
+import os
+import pwd
+import time
+import json
+import argparse
+import psutil
+from time import strftime
+from datetime import datetime, timedelta
+
+
+def parseArguments():
+  """
+  Parse arguments for monitor collector instance.
+  """
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--output_folder',
+                      help='Path of the folder where output files should be written.')
+  parser.add_argument('--partition_id',
+                      help='ID of the computer partition to collect data from.')
+  parser.add_argument('--collector_db',
+                      help='The path of slapos collect database.')
+
+  return parser.parse_args()
+
+
+class RessourceCollect:
+
+  def __init__(self, db_path = None):
+    assert os.path.exists(db_path) and os.path.isfile(db_path)
+    self.uri = db_path
+    self.connection = None
+    self.cursor = None
+
+  def connect(self):
+    self.connection = sqlite3.connect(self.uri)
+    self.cursor = self.connection.cursor()
+
+  def close(self):
+    assert self.connection is not None
+    self.cursor.close()
+    self.connection.close()
+
+  def _execute(self, sql):
+    assert self.connection is not None
+    return self.cursor.execute(sql)
+
+  def select(self, table, date=None, columns="*", where=None):
+    """ Query database for a full table information """
+    if date is not None:
+      where_clause = " WHERE date = '%s' " % date
+    else:
+      where_clause = ""
+    
+    if where is not None:
+      if where_clause == "":
+        where_clause += " WHERE 1 = 1 "
+      where_clause += " AND %s " % where 
+    select_sql = "SELECT %s FROM %s %s " % (columns, table, where_clause)
+    return self._execute(select_sql)
+
+  def has_table(self, name):
+    self.connect()
+    check_result_cursor = self.select(
+      table="sqlite_master",
+      columns='name',
+      where="type='table' AND name='%s'" % name)
+    table_exists_result = zip(*check_result_cursor)
+    if not len(table_exists_result) or table_exists_result[0][0] is None:
+      return False
+    return True
+
+  def getPartitionCPULoadAverage(self, partition_id, date_scope):
+    self.connect()
+    query_result_cursor = self.select("user", date_scope,
+                       columns="SUM(cpu_percent)", 
+                       where="partition = '%s'" % partition_id)
+
+    cpu_percent_sum = zip(*query_result_cursor)
+    if len(cpu_percent_sum) and cpu_percent_sum[0][0] is None:
+      return
+
+    query_result_cursor = self.select("user", date_scope,
+                       columns="COUNT(DISTINCT time)", 
+                       where="partition = '%s'" % partition_id)
+
+    sample_amount = zip(*query_result_cursor)
+    self.close()
+
+    if len(sample_amount) and len(cpu_percent_sum):
+      return round(cpu_percent_sum[0][0]/sample_amount[0][0], 2)
+
+  def getPartitionUsedMemoryAverage(self, partition_id, date_scope):
+    self.connect()
+    query_result_cursor = self.select("user", date_scope,
+                       columns="SUM(memory_rss)", 
+                       where="partition = '%s'" % partition_id)
+
+    memory_sum = zip(*query_result_cursor)
+    if len(memory_sum) and memory_sum[0][0] is None:
+      return
+
+    query_result_cursor = self.select("user", date_scope,
+                       columns="COUNT(DISTINCT time)", 
+                       where="partition = '%s'" % partition_id)
+
+    sample_amount = zip(*query_result_cursor)
+    self.close()
+
+    if len(sample_amount) and len(memory_sum):
+      return round(memory_sum[0][0]/(sample_amount[0][0]*1024*1024.0), 2)
+
+  def getPartitionDiskUsedAverage(self, partition_id, date_scope):
+    if not self.has_table('folder'):
+      return
+    self.db.connect()
+    query_result_cursor = self.select("folder", date_scope,
+                       columns="SUM(disk_used)", 
+                       where="partition = '%s'" % partition_id)
+
+    disk_used_sum = zip(*query_result_cursor)
+    if len(disk_used_sum) and disk_used_sum[0][0] is None:
+      return
+    query_result_cursor = self.select("folder", date_scope,
+                       columns="COUNT(DISTINCT time)", 
+                       where="partition = '%s'" % partition_id)
+  
+    collect_amount = zip(*query_result_cursor)
+    self.db.close()
+  
+    if len(collect_amount) and len(disk_used_sum):
+      return round(disk_used_sum[0][0]/(collect_amount[0][0]*1024.0), 2)
+
+
+  def getPartitionConsumption(self, partition_id, where=""):
+    """
+      Query collector db to get consumed ressource for last minute
+    """
+    self.connect()
+    comsumption_list = []
+    if where != "":
+      where = "and %s" % where
+    date_scope = datetime.now().strftime('%Y-%m-%d')
+    min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00')
+    max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59')
+
+    sql_query = """select count(pid), SUM(cpu_percent) as cpu_result, SUM(cpu_time), 
+MAX(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), pid, SUM(io_rw_counter), 
+SUM(io_cycles_counter) from user 
+where date='%s' and partition='%s' and (time between '%s' and '%s') %s 
+group by pid order by cpu_result desc""" % (
+        date_scope, partition_id, min_time, max_time, where)
+    query_result = self._execute(sql_query)
+    for result in query_result:
+      count = int(result[0])
+      if not count > 0:
+        continue
+      resource_dict = {
+        'pid': result[6],
+        'cpu_percent': round(result[1]/count, 2),
+        'cpu_time': round((result[2] or 0)/(60.0), 2),
+        'cpu_num_threads': round(result[3]/count, 2),
+        'memory_percent': round(result[4]/count, 2),
+        'memory_rss': round((result[5] or 0)/(1024*1024.0), 2),
+        'io_rw_counter': round(result[7]/count, 2),
+        'io_cycles_counter': round(result[8]/count, 2)
+      }
+      try:
+        pprocess = psutil.Process(int(result[6]))
+      except psutil.NoSuchProcess:
+        pass
+      else:
+        resource_dict['name'] = pprocess.name()
+        resource_dict['command'] = pprocess.cmdline()
+        resource_dict['user'] = pprocess.username()
+        resource_dict['date'] = datetime.fromtimestamp(pprocess.create_time()).strftime("%Y-%m-%d %H:%M:%S")
+      comsumption_list.append(resource_dict)
+    self.close()
+    return comsumption_list
+  
+  def getPartitionComsumptionStatus(self, partition_id, where=""):
+    self.connect()
+    if where != "":
+      where = " and %s" % where
+    date_scope = datetime.now().strftime('%Y-%m-%d')
+    min_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:00')
+    max_time = (datetime.now() - timedelta(minutes=1)).strftime('%H:%M:59')
+    sql_query = """select count(pid), SUM(cpu_percent), SUM(cpu_time), 
+SUM(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), SUM(io_rw_counter), 
+SUM(io_cycles_counter) from user where 
+date='%s' and partition='%s' and (time between '%s' and '%s') %s""" % (
+        date_scope, partition_id, min_time, max_time, where)
+    query_result = self._execute(sql_query)
+    result_list = zip(*query_result)
+
+    process_dict = memory_dict = io_dict = {}
+    if len(result_list):
+      result = result_list
+      process_dict = {'total_process': result[0][0],
+        'cpu_percent': round((result[1][0] or 0), 2),
+        'cpu_time': round((result[2][0] or 0)/(60.0), 2),
+        'cpu_num_threads': round((result[3][0] or 0), 2),
+        'date': '%s %s' % (date_scope, min_time)
+      }
+      memory_dict = {'memory_percent': round((result[4][0] or 0), 2),
+        'memory_rss': round((result[5][0] or 0)/(1024*1024.0), 2),
+        'date': '%s %s' % (date_scope, min_time)
+      }
+      io_dict = {'io_rw_counter': round((result[6][0] or 0), 2),
+        'io_cycles_counter': round((result[7][0] or 0), 2),
+        'disk_used': 0,
+        'date': '%s %s' % (date_scope, min_time)
+      }
+      if self.has_table('folder'):
+        disk_result_cursor = self.select(
+          "folder", date_scope,
+          columns="SUM(disk_used)", 
+          where="partition='%s' and (time between '%s' and '%s') %s" % (
+            partition_id, min_time, max_time, where
+          )
+        )
+    
+        disk_used_sum = zip(*query_result_cursor)
+        if len(disk_used_sum) and disk_used_sum[0][0] is not None:
+          io_dict['disk_used'] = round(disk_used_sum[0][0]/1024.0, 2)
+    self.close()
+    return (process_dict, memory_dict, io_dict)
+
+def appendToJsonFile(file_path, content, stepback=2):
+  with open (file_path, mode="r+") as jfile:
+    jfile.seek(0, 2)
+    position = jfile.tell() - stepback
+    jfile.seek(position)
+    jfile.write('%s}' % ',"{}"]'.format(content))
+
+if __name__ == "__main__":
+  parser = parseArguments()
+  if not os.path.exists(parser.output_folder) and os.path.isdir(parser.output_folder):
+    raise Exception("Invalid ouput folder: %s" % parser.output_folder)
+  collector = RessourceCollect(parser.collector_db)
+
+  date_scope = datetime.now().strftime('%Y-%m-%d')
+  stat_info = os.stat(parser.output_folder)
+  partition_user = pwd.getpwuid(stat_info.st_uid)[0]
+
+  # Consumption global status
+  process_file = os.path.join(parser.output_folder, 'monitor_resource_process.data.json')
+  mem_file = os.path.join(parser.output_folder, 'monitor_resource_memory.data.json')
+  io_file = os.path.join(parser.output_folder, 'monitor_resource_io.data.json')
+
+  process_result, memory_result, io_result = collector.getPartitionComsumptionStatus(partition_user)
+  resource_file = os.path.join(parser.output_folder, 'monitor_process_resource.status.json')
+
+  label_list = ['date', 'total_process', 'cpu_percent', 'cpu_time', 'cpu_num_threads',
+                  'memory_percent', 'memory_rss', 'io_rw_counter', 'io_cycles_counter',
+                  'disk_used']
+  resource_status_dict = {}
+  if not os.path.exists(process_file):
+    with open(process_file, 'w') as fprocess:
+      data_dict = {
+        "date": time.time(),
+        "data": ["date, total process, CPU percent, CPU time, CPU threads"]
+      }
+      fprocess.write(json.dumps(data_dict))
+
+  if not os.path.exists(mem_file):
+    with open(mem_file, 'w') as fmem:
+      data_dict = {
+        "date": time.time(),
+        "data": ["date, memory used percent, memory used"]
+      }
+      fmem.write(json.dumps(data_dict))
+
+  if not os.path.exists(io_file):
+    with open(io_file, 'w') as fio:
+      data_dict = {
+        "date": time.time(),
+        "data": ["date, io rw counter, io cycles counter, disk used"]
+      }
+      fio.write(json.dumps(data_dict))
+
+  if process_result and process_result['total_process'] != 0.0:
+    appendToJsonFile(process_file, ", ".join(
+      [str(process_result[key]) for key in label_list if process_result.has_key(key)])
+    )
+    resource_status_dict.update(process_result)
+  if memory_result and memory_result['memory_rss'] != 0.0:
+    appendToJsonFile(mem_file, ", ".join(
+      [str(memory_result[key]) for key in label_list if memory_result.has_key(key)])
+    )
+    resource_status_dict.update(memory_result)
+  if io_result and io_result['io_rw_counter'] != 0.0:
+    appendToJsonFile(io_file, ", ".join(
+      [str(io_result[key]) for key in label_list if io_result.has_key(key)])
+    )
+    resource_status_dict.update(io_result)
+
+  with open(os.path.join(parser.output_folder, 'monitor_resource.status.json'), 'w') as fp:
+    fp.write(json.dumps(resource_status_dict))
+
+  # Consumption Ressource
+  resource_process_status_list = collector.getPartitionConsumption(partition_user)
+  if resource_process_status_list:
+    with open(resource_file, 'w') as rf:
+      rf.write(json.dumps(resource_process_status_list))

stack/monitor2/scripts/globalstate.py

+#!/usr/bin/env python
+
+import sys
+import os
+import glob
+import json
+import ConfigParser
+import time
+
+def softConfigGet(config, *args, **kwargs):
+  try:
+    return config.get(*args, **kwargs)
+  except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
+    return ""
+
+def generateStatisticsData(stat_file_path, content):
+  # csv document for statictics
+  if not os.path.exists(stat_file_path):
+    with open(stat_file_path, 'w') as fstat:
+      data_dict = {
+        "date": time.time(),
+        "data": ["Date, Success, Error, Warning"]
+      }
+      fstat.write(json.dumps(data_dict))
+
+  current_state = ''
+  if content.has_key('state'):
+    current_state = '%s, %s, %s, %s' % (
+      content['date'],
+      content['state']['success'],
+      content['state']['error'],
+      content['state']['warning'])
+
+  # append to file
+  if current_state:
+    with open (stat_file_path, mode="r+") as fstat:
+      fstat.seek(0,2)
+      position = fstat.tell() -2
+      fstat.seek(position)
+      fstat.write('%s}' % ',"{}"]'.format(current_state))
+
+def main(args_list):
+  monitor_file, instance_file = args_list
+
+  monitor_config = ConfigParser.ConfigParser()
+  monitor_config.read(monitor_file)
+
+  base_folder = monitor_config.get('monitor', 'public-folder')
+  base_url = monitor_config.get('monitor', 'base-url')
+  related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
+  statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
+
+  if not os.path.exists(statistic_folder):
+    try:
+      os.makedirs(statistic_folder)
+    except OSError, e:
+      if e.errno == os.errno.EEXIST and os.path.isdir(statistic_folder):
+        pass
+      else: raise
+
+  # search for all status files
+  file_list = filter(os.path.isfile,
+      glob.glob("%s/*.status.json" % base_folder)
+    )
+  error = warning = success = 0
+  latest_date = ''
+  status = 'OK'
+  promise_list = []
+  global_state_file = os.path.join(base_folder, 'monitor.global.json')
+  for file in file_list:
+    try:
+      with open(file, 'r') as temp_file:
+        tmp_json = json.loads(temp_file.read())
+    except ValueError:
+      # bad json file ?
+      continue
+    if tmp_json['status'] == 'ERROR':
+      error  += 1
+    elif tmp_json['status'] == 'OK':
+      success += 1
+    elif tmp_json['status'] == 'WARNING':
+      warning += 1
+    if tmp_json['start-date'] > latest_date:
+      latest_date = tmp_json['start-date']
+    tmp_json['time'] = tmp_json['start-date'].split(' ')[1]
+    del tmp_json['start-date']
+    promise_list.append(tmp_json)
+
+  if error:
+    status = 'ERROR'
+  elif warning:
+    status = 'WARNING'
+
+  global_state_dict = dict(
+      status=status,
+      state={
+        'error': error,
+        'success': success,
+        'warning': warning,
+      },
+      date=latest_date,
+      _links={"rss_url": {"href": "%s/public/feed" % base_url},
+              "public_url": {"href": "%s/share/jio_public" % base_url},
+              "private_url": {"href": "%s/share/jio_private" % base_url},
+              "data_url": {"href": '%s/public/data' % base_url}
+            },
+      data={'state': 'monitor_state.data',
+            'process_state': 'monitor_process_resource.status',
+            'process_resource': 'monitor_resource_process.data',
+            'memory_resource': 'monitor_resource_memory.data',
+            'io_resource': 'monitor_resource_io.data',
+            'monitor_process_state': 'monitor_resource.status'}
+    )
+
+  global_state_dict['_embedded'] = {'promises': promise_list}
+
+  if os.path.exists(instance_file):
+    config = ConfigParser.ConfigParser()
+    config.read(instance_file)
+    if 'instance' in config.sections():
+      instance_dict = {}
+      global_state_dict['title'] = config.get('instance', 'name')
+      if not global_state_dict['title']:
+        global_state_dict['title'] = 'Instance Monitoring'
+      
+      instance_dict['computer'] = config.get('instance', 'computer')
+      instance_dict['ipv4'] = config.get('instance', 'ipv4')
+      instance_dict['ipv6'] = config.get('instance', 'ipv6')
+      instance_dict['software-release'] = config.get('instance', 'software-release')
+      instance_dict['software-type'] = config.get('instance', 'software-type')
+      instance_dict['partition'] = config.get('instance', 'partition')
+
+      global_state_dict['_embedded'].update({'instance' : instance_dict})
+
+  if related_monitor_list:
+    global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/jio_public" % url}
+                          for url in related_monitor_list]
+
+  with open(global_state_file, 'w') as fglobal:
+    fglobal.write(json.dumps(global_state_dict))
+
+  generateStatisticsData(
+    os.path.join(statistic_folder, 'monitor_state.data.json'),
+    global_state_dict)
+
+  return 0
+
+if __name__ == "__main__":
+  if len(sys.argv) < 3:
+    print("Usage: %s <monitor_conf_path> <instance_conf_path>" % sys.argv[0])
+    sys.exit(2)
+  sys.exit(main(sys.argv[1:]))

stack/monitor2/scripts/monitor.py

@@ -3,12 +3,30 @@
 import sys
 import os
 import stat
-import subprocess
-import threading
 import json
 import ConfigParser
 import traceback
 import argparse
+import time
+import glob
+from datetime import datetime
+
+OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
+<!-- OPML generated by SlapOS -->
+<opml version="1.1">
+	<head>
+		<title>SlapOS Monitoring Status Lists</title>
+		<dateCreated>%(creation_date)s</dateCreated>
+		<dateModified>%(modification_date)s</dateModified>
+	</head>
+	<body>
+	  <outline text="%(outline_title)s">"""
+OPML_END = """	  </outline>
+  </body>
+</opml>"""
+
+OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" />'
+
 
 def parseArguments():
   """
@@ -64,14 +82,22 @@ class Monitoring(object):
     self.title = config.get("monitor", "title")
     self.service_pid_folder = config.get("monitor", "service-pid-folder")
     self.crond_folder = config.get("monitor", "crond-folder")
+    self.logrotate_d = config.get("monitor", "logrotate-folder")
     self.wraper_folder = config.get("monitor", "wraper-folder")
     self.promise_runner = config.get("monitor", "promise-runner")
     self.promise_folder_list = config.get("monitor", "promise-folder-list").split()
     self.public_folder = config.get("monitor", "public-folder")
     self.private_folder = config.get("monitor", "private-folder")
+    self.collector_db  = config.get("monitor", "collector-db")
+    self.collect_script = config.get("monitor", "collect-script")
+    self.webdav_folder = config.get("monitor", "webdav-folder")
     self.public_path_list = config.get("monitor", "public-path-list").split()
     self.private_path_list = config.get("monitor", "private-path-list").split()
     self.monitor_url_list = config.get("monitor", "monitor-url-list").split()
+    self.webdav_url = '%s/share' % config.get("monitor", "base-url")
+    self.public_url = '%s/public' % config.get("monitor", "base-url")
+    self.status_history_folder = os.path.join(self.public_folder, 'history')
+    self.python = config.get("monitor", "python") or "python"
 
     self.promise_dict = {}
     for promise_folder in self.promise_folder_list:
@@ -101,11 +127,12 @@ class Monitoring(object):
     #    if promise_name in promise_dict:
     #      loadConfig([path], promise_dict[promise_name]["configuration"])
 
-  def createSymlinksFromConfig(self, destination_folder, source_path_list, service_name=""):
+  def createSymlinksFromConfig(self, destination_folder, source_path_list, name=""):
     if destination_folder:
       if source_path_list:
         for path in source_path_list:
-          dirname = os.path.join(destination_folder, service_name)
+          path = path.rstrip('/')
+          dirname = os.path.join(destination_folder, name)
           try:
             mkdirAll(dirname)  # could also raise OSError
             os.symlink(path, os.path.join(dirname, os.path.basename(path)))
@@ -113,27 +140,93 @@ class Monitoring(object):
             if e.errno != os.errno.EEXIST:
               raise
 
+  def configureFolders(self):
+    # configure public and private folder
+    self.createSymlinksFromConfig(self.webdav_folder, [self.public_folder])
+    self.createSymlinksFromConfig(self.webdav_folder, [self.private_folder])
+
+    #configure jio_documents folder
+    jio_public = os.path.join(self.webdav_folder, 'jio_public')
+    jio_private = os.path.join(self.webdav_folder, 'jio_private', '.jio_documents')
+    mkdirAll(jio_public)
+    mkdirAll(jio_private)
+    mkdirAll(self.status_history_folder)
+    try:
+      os.symlink(self.public_folder, os.path.join(jio_public, '.jio_documents'))
+    except OSError, e:
+      if e.errno != os.errno.EEXIST:
+        raise
+    try:
+      os.symlink(self.private_folder, os.path.join(jio_private, '.jio_documents'))
+    except OSError, e:
+      if e.errno != os.errno.EEXIST:
+        raise
+
+  def generateOpmlFile(self, feed_url_list, output_file):
+    if not feed_url_list:
+      return
+
+    if os.path.exists(output_file):
+      creation_date = datetime.fromtimestamp(os.path.getctime(output_file)).utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+      modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+    else:
+      creation_date = modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+
+    opml_content = OPML_START % {'creation_date': creation_date,
+                                  'modification_date': modification_date,
+                                  'outline_title': 'Monitoring RSS Feed list'}
+
+    opml_content += OPML_OUTLINE_FEED % {'title': 'Main Instance',
+        'html_url': self.public_url + '/feed',
+        'xml_url': self.public_url + '/feed'}
+    for feed_url in feed_url_list:
+      opml_content += OPML_OUTLINE_FEED % {'title': 'Monitoring Instance',
+        'html_url': feed_url + '/public/feed',
+        'xml_url': feed_url + '/public/feed'}
+
+    opml_content += OPML_END
+
+    with open(output_file, 'w') as wfile:
+      wfile.write(opml_content)
+
+  def generateLogrotateEntry(self, name, file_list, option_list):
+    """
+      Will add a new entry in logrotate.d folder. This can help to rotate data file daily
+    """
+    content = "%(logfiles)s {\n%(options)s\n}\n" % {
+                'logfiles': ' '.join(file_list),
+                'options': '\n'.join(option_list)
+              }
+    file_path = os.path.join(self.logrotate_d, name)
+    with open(file_path, 'w') as flog:
+      flog.write(content)
+
   def generateMonitorHalJson(self):
+    monitor_link_dict = {"webdav": {"href": self.webdav_url},
+                          "public": {"href": "%s/public" % self.webdav_url},
+                          "private": {"href": "%s/private" % self.webdav_url},
+                          "rss": {"href": "%s/feed" % self.public_url},
+                          "jio_public": {"href": "%s/jio_public" % self.webdav_url},
+                          "jio_private": {"href": "%s/jio_private" % self.webdav_url}
+                        }
     if self.title:
       self.monitor_dict["title"] = self.title
     if self.monitor_url_list:
-      self.monitor_dict["_links"] = {"related_monitor": [{"href": url}
-                                  for url in self.monitor_url_list]}
+      monitor_link_dict["related_monitor"] = [{"href": "%s/share/jio_public" % url}
+                                  for url in self.monitor_url_list]
+      self.generateOpmlFile(self.monitor_url_list,
+        os.path.join(self.public_folder, 'feeds'))
+    self.monitor_dict["_links"] = monitor_link_dict
     if self.promise_items:
       service_list = []
       for service_name, promise in self.promise_items:
         service_config = promise["configuration"]
+        tmp = softConfigGet(service_config, "service", "title")
         service_dict = {}
         service_dict["id"] = service_name
-        service_dict["_links"] = {"status": {"href": "/public/%s.status.json" % service_name}}  # hardcoded
-        tmp = softConfigGet(service_config, "service", "title")
+        service_dict["_links"] = {"status": {"href": "%s/public/%s.status.json" % (self.webdav_url, service_name)}}  # hardcoded
         if tmp:
           service_dict["title"] = tmp
-        interface_path = os.path.join(self.private_folder, service_name, "interface/index.html")  # hardcoded
-        if os.path.isfile(interface_path):
-          service_dict["_links"]["interface"] = {"href": "/private/%s/interface/" % service_name}  # hardcoded
-        else:
-          service_dict["_links"]["interface"] = {"href": "/default-promise-interface.html?service_name=%s" % service_name}  # XXX hardcoded
         service_list.append(service_dict)
 
       self.monitor_dict["_embedded"] = {"service": service_list}
@@ -143,27 +236,58 @@ class Monitoring(object):
 
   def generateServiceCronEntries(self):
     # XXX only if at least one configuration file is modified, then write in the cron
-    cron_line_list = []
+    cron_line_list = ['PATH=%s\n' % os.environ['PATH']]
+
+    service_name_list = [name.replace('.status.json', '')
+      for name in os.listdir(self.public_folder) if name.endswith('.status.json')]
+
     for service_name, promise in self.promise_items:
       service_config = promise["configuration"]
       service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)  # hardcoded
       mkdirAll(os.path.dirname(service_status_path))
-      command = ("%s %s %s " % (
+      command = "%s %s %s %s %s %s %s" % (
           self.promise_runner,
           os.path.join(self.service_pid_folder, "%s.pid" % service_name),
-          service_status_path,)
-        ) + promise["path"]
+          service_status_path,
+          promise["path"],
+          service_name,
+          "%s/jio_public" % self.webdav_url, # XXX hardcoded
+          self.status_history_folder
+        )
+
       cron_line_list.append("%s %s" % (
           softConfigGet(service_config, "service", "frequency") or "* * * * *",
           command.replace("%", "\\%"),
         ))
+
+      if service_name in service_name_list:
+        service_name_list.pop(service_name_list.index(service_name))
+
       wrapper_path = os.path.join(self.wraper_folder, service_name)
       with open(wrapper_path, "w") as fp:
         fp.write("#!/bin/sh\n%s" % command)  # XXX hardcoded, use dash, sh or bash binary!
       os.chmod(wrapper_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH )
+
+    if service_name_list != []:
+      # XXX Some service was removed, delete his status file so monitor will not consider his status anymore
+      for service_name in service_name_list:
+        status_path = os.path.join(self.public_folder, '%s.status.json' % service_name)
+        if os.path.exists(status_path):
+          try:
+            os.unlink(status_path)
+          except OSError, e:
+            print "Error: Failed to delete %s" % status_path, str(e)
+            pass
+
     with open(self.crond_folder + "/monitor-promises", "w") as fp:
       fp.write("\n".join(cron_line_list))
 
+  def addCronEntry(self, name, frequency, command):
+    entry_line = '%s %s' % (frequency, command)
+    cron_entry_file = os.path.join(self.crond_folder, name)
+    with open(cron_entry_file, "w") as cronf:
+      cronf.write(entry_line)
+
   def bootstrapMonitor(self):
     # create symlinks from service configurations
     self.promise_items = self.promise_dict.items()
@@ -184,6 +308,8 @@ class Monitoring(object):
     self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
     self.createSymlinksFromConfig(self.private_folder, self.private_path_list)
 
+    self.configureFolders()
+
     # generate monitor.json
     self.monitor_dict = {}
     self.generateMonitorHalJson()
@@ -191,6 +317,22 @@ class Monitoring(object):
     # put promises to a cron file
     self.generateServiceCronEntries()
 
+    # Rotate monitor data files
+    self.data_folder = os.path.join(self.public_folder, 'data', '.jio_documents')
+    mkdirAll(self.data_folder)
+    option_list = [
+      'daily', 'nocreate', 'noolddir', 'rotate 30',
+      'nocompress', 'extension .json', 'dateext',
+      'dateformat -%Y-%m-%d', 'notifempty'
+    ]
+    file_list = ["%s/*.data.json" % self.data_folder]
+    self.generateLogrotateEntry('monitor.data', file_list, option_list)
+
+    # Add cron entry for SlapOS Collect
+    command = "%s %s --output_folder %s --collector_db %s" % (self.python,
+      self.collect_script, self.data_folder, self.collector_db)
+    self.addCronEntry('monitor_collect', '* * * * *', command)
+
     return 0
 
 

stack/monitor2/scripts/rsstoopml.py

+#!/usr/bin/env python
+
+import json
+import os
+import time
+from datetime import datetime
+
+OPML_START = """<?xml version="1.0" encoding="UTF-8"?>
+<!-- OPML generated by SlapOS -->
+<opml version="1.1">
+	<head>
+		<title>SlapOS Monitoring Status Lists</title>
+		<dateCreated>%(creation_date)s</dateCreated>
+		<dateModified>%(mondification_date)s</dateModified>
+	</head>
+	<body>
+	  <outline text="%(outline_title)s">"""
+OPML_END = """	  </outline>
+  </body>
+</opml>"""
+
+OPML_OUTLINE_FEED = '<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(html_url)s" xmlUrl="%(xml_url)s" />'
+
+def main(config_file, output_file):
+  feed_url_list = []
+  if os.path.exists(output_file):
+    creation_date = datetime.fromtimestamp(os.path.getctime(output_file)).utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+    modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+  else:
+    creation_date = modification_date = datetime.utcnow().strftime("%a, %d %b %Y %H:%M:%S +0000")
+  with open(config_file, 'r') as fconfig:
+    feed_url_list = json.loads(fconfig.read())
+
+  opml_content = OPML_START
+
+	for feed_line in feed_url_list:
+		opml_content += OPML_OUTLINE_FEED % {'title': feed_line['title'], 'html_url': feed_line['url'], 'xml_url': feed_line['url']}
+
+	opml_content += OPML_END
+
+  with open(output_file, 'w') as wfile:
+    wfile.write(opml_content)
+
+if __name__ == "__main__":
+  if len(sys.argv) < 3:
+    print("Usage: %s <rss_conf_file> <output_path>" % sys.argv[0])
+    sys.exit(2)
+	config_file = sys.argv[1]
+	output_file = sys.argv[2]
+	main(config_file, output_file)
\ No newline at end of file

stack/monitor2/scripts/run-promise.py

-#!/usr/bin/env python
+#!{{ python }}
 # -*- coding: utf-8 -*-
 
 import sys
 import os
 import subprocess
 import json
+import psutil
+import time
+from shutil import copyfile
+import glob
 
 def main():
   if len(sys.argv) < 4:
-    print("Usage: %s <pid_path> <output_path> <command...>" % sys.argv[0])
+    print("Usage: %s <pid_path> <output_path> <command> [<name>] [...]" % sys.argv[0])
     return 2
   pid_path=sys.argv[1]
   output_path=sys.argv[2]
+  promise_name = history_folder = related_url = ""
+  if len(sys.argv) >= 5:
+    promise_name = sys.argv[4]
+  if len(sys.argv) >= 6:
+    related_url = sys.argv[5]
+  if len(sys.argv) >= 7:
+    history_folder = sys.argv[6]
   if os.path.exists(pid_path):
     with open(pid_path, "r") as pidfile:
       try:
@@ -21,27 +32,82 @@ def main():
       if pid and os.path.exists("/proc/" + str(pid)):
         print("A process is already running with pid " + str(pid))
         return 1
+  start_date = ""
   with open(pid_path, "w") as pidfile:
     process = executeCommand(sys.argv[3:])
+    ps_process = psutil.Process(process.pid)
+    start_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ps_process.create_time()))
     pidfile.write(str(process.pid))
-  status_json = generateStatusJsonFromProcess(process)
+  
+  status_json = generateStatusJsonFromProcess(process, start_date=start_date, title=promise_name)
+
+  # Save the lastest status change date (needed for rss)
+  if related_url:
+    status_json['_links'] = {"monitor": {"href": related_url}}
+  status_json['change-time'] = ps_process.create_time()
+  if os.path.exists(output_path):
+    with open(output_path) as f:
+      last_result = json.loads(f.read())
+      if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
+        status_json['change-time'] = last_result['change-time']
+
+  if history_folder:
+    updateStatusHistoryFolder(promise_name, output_path, history_folder)
   with open(output_path, "w") as outputfile:
     json.dump(status_json, outputfile)
   os.remove(pid_path)
 
+def updateStatusHistoryFolder(name, status_file, history_folder):
+  old_history_list = []
+  history_path = os.path.join(history_folder, name, '.jio_documents')
+  if not os.path.exists(status_file):
+    return
+  if not os.path.exists(history_folder):
+    return
+  if not os.path.exists(history_path):
+    try:
+      os.makedirs(history_path)
+    except OSError, e:
+      if e.errno == os.errno.EEXIST and os.path.isdir(history_path):
+        pass
+      else: raise
+  with open(status_file, 'r') as sf:
+    status_dict = json.loads(sf.read())
+    filename = '%s.status.json' % (
+      status_dict['start-date'].replace(' ', '_').replace(':', ''))
+
+  copyfile(status_file, os.path.join(history_path, filename))
+  # Don't let history foler grow too much, keep 30 files
+  file_list = filter(os.path.isfile,
+      glob.glob("%s/*.status.json" % history_path)
+    )
+  file_count = len(file_list)
+  if file_count > 30:
+    file_list.sort(key=lambda x: os.path.getmtime(x))
+    while file_count > 30:
+      to_delete = file_list.pop(0)
+      try:
+        os.unlink(to_delete)
+        file_count -= 1
+      except OSError:
+        raise
 
-def generateStatusJsonFromProcess(process):
+def generateStatusJsonFromProcess(process, start_date=None, title=None):
   stdout, stderr = process.communicate()
   try:
     status_json = json.loads(stdout)
   except ValueError:
     status_json = {}
   if process.returncode != 0:
-    status_json["status"] = "error"
+    status_json["status"] = "ERROR"
   elif not status_json.get("status"):
     status_json["status"] = "OK"
   if stderr:
-    status_json["error"] = stderr
+    status_json["message"] = stderr
+  if start_date:
+    status_json["start-date"] = start_date
+  if title:
+    status_json["title"] = title
   return status_json
 
 

stack/monitor2/scripts/status2rss.py

@@ -4,104 +4,47 @@ import json
 import datetime
 import base64
 import hashlib
+import PyRSS2Gen
+
+def getKey(item):
+  return item.pubDate
 
 def main():
-  _, title, link, public_folder, previous_status_path, output_path = sys.argv
-  final_status = "OK";
-  # getting status
-  for filename in os.listdir(public_folder):
+  _, title, link, base_url, status_folder, output_path = sys.argv
+
+  rss_item_list = []
+  for filename in os.listdir(status_folder):
     if filename.endswith(".status.json"):
-      filepath = os.path.join(public_folder, filename)
-      status = None
+      filepath = os.path.join(status_folder, filename)
+      result_dict = None
       try:
-        status = json.load(open(filepath, "r"))
+        result_dict = json.load(open(filepath, "r"))
       except ValueError:
+        print "Failed to load json file: %s" % filepath
         continue
-      try:
-        if status["status"] != "OK":
-          final_status = "BAD"
-          break
-      except KeyError:
-        final_status = "BAD"
-        break
-  # checking previous status
-  try:
-    status = open(previous_status_path, "r").readline(4)
-    if status == final_status:
-      return 0
-  except IOError:
-    pass
-  # update status
-  open(previous_status_path, "w").write(final_status)
-  # generating RSS
-  utcnow = datetime.datetime.utcnow()
-  open(output_path, "w").write(
-    newRssString(
-      title,
-      title,
-      link,
-      utcnow,
-      utcnow,
-      "60",
-      [
-        newRssItemString(
-          "Status is %s" % final_status,
-          "Status is %s" % final_status,
-          link,
-          newGuid("%s, %s" % (utcnow, final_status)),
-          utcnow,
-        )
-      ],
+      description = result_dict.get('message', '')
+      event_time = datetime.datetime.fromtimestamp(result_dict['change-time'])
+      rss_item = PyRSS2Gen.RSSItem(
+        title = '[%s] %s' % (result_dict['status'], result_dict['title']),
+        description = "%s: %s\n%s" % (event_time, result_dict['status'], description),
+        link = '%s/%s' % (base_url, filename),
+        pubDate = event_time,
+        guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s" % (event_time, result_dict['status'])))
       )
-  )
-
-
-def escapeHtml(string):
-  return string.replace("&", "&amp;").replace("<", "&lt;").replace(">", "&gt;").replace("\"", "&quot;")
+      rss_item_list.append(rss_item)
 
-def newGuid(string):
-  sha256 = hashlib.sha256()
-  sha256.update(string)
-  return sha256.hexdigest()
 
-def newRssItemString(title, description, link, guid, pub_date, guid_is_perma_link=True):
-  return """<item>
- <title>%(title)s</title>
- <description>%(description)s</description>
- <link>%(link)s</link>
- <guid isPermaLink="%(guid_is_perma_link)s">%(guid)s</guid>
- <pubDate>%(pub_date)s</pubDate>
-</item>""" % {
-    "title": escapeHtml(title),
-    "description": escapeHtml(description),
-    "link": escapeHtml(link),
-    "guid": escapeHtml(guid),
-    "pub_date": escapeHtml(pub_date.strftime("%a, %d %b %Y %H:%M:%S +0000")),
-    "guid_is_perma_link": escapeHtml(repr(guid_is_perma_link).lower()),
-  }
-
-def newRssString(title, description, link, last_build_date, pub_date, ttl, rss_item_string_list):
-  return """<?xml version="1.0" encoding="UTF-8" ?>
-<rss version="2.0">
-<channel>
- <title>%(title)s</title>
- <description>%(description)s</description>
- <link>%(link)s</link>
- <lastBuildDate>%(last_build_date)s</lastBuildDate>
- <pubDate>%(pub_date)s</pubDate>
- <ttl>%(ttl)s</ttl>
-%(items)s
-</channel>
-</rss>
-""" % {
-    "title": escapeHtml(title),
-    "description": escapeHtml(description),
-    "link": escapeHtml(link),
-    "last_build_date": escapeHtml(last_build_date.strftime("%a, %d %b %Y %H:%M:%S +0000")),
-    "pub_date": escapeHtml(pub_date.strftime("%a, %d %b %Y %H:%M:%S +0000")),
-    "ttl": escapeHtml(str(ttl)),
-    "items": "\n\n".join([" " + item.replace("\n", "\n ") for item in rss_item_string_list]),
-  }
+  ### Build the rss feed
+  sorted(rss_item_list, key=getKey)
+  rss_feed = PyRSS2Gen.RSS2 (
+    title = title,
+    link = link,
+    description = '',
+    lastBuildDate = datetime.datetime.utcnow(),
+    items = rss_item_list
+    )
+  with open(output_path, 'w') as frss:
+    frss.write(rss_feed.to_xml())
 
 if __name__ == "__main__":
   exit(main())

stack/monitor2/templates/instance-info.conf.in

+[instance]
+name = {{ instance_dict['name'] }}
+computer = {{ instance_dict['computer-id'] }}
+ipv4 = {{ instance_dict['ipv4'] }}
+ipv6 = {{ instance_dict['ipv6'] }}
+software-release = {{ instance_dict['software-release'] }}
+software-type = {{ instance_dict['software-type'] }}
+partition = {{ instance_dict['partition-id'] }}
\ No newline at end of file

stack/monitor2/templates/monitor-httpd.conf.in

@@ -11,7 +11,7 @@ ServerAdmin someone@email
 Listen [{{ parameter_dict.get('listening-ip') }}]:{{ parameter_dict.get('port') }}
 Define MonitorPort
 </IfDefine>
-DocumentRoot "{{ directory.get('www') }}"
+DocumentRoot "{{ directory.get('webdav') }}"
 ErrorLog "{{ parameter_dict.get('error-log') }}"
 LoadModule unixd_module modules/mod_unixd.so
 LoadModule access_compat_module modules/mod_access_compat.so
@@ -31,6 +31,8 @@ LoadModule proxy_module      modules/mod_proxy.so
 LoadModule proxy_http_module modules/mod_proxy_http.so
 LoadModule rewrite_module modules/mod_rewrite.so
 LoadModule headers_module modules/mod_headers.so
+LoadModule dav_module modules/mod_dav.so
+LoadModule dav_fs_module modules/mod_dav_fs.so
 
 # SSL Configuration
 <IfDefine !SSLConfigured>
@@ -49,6 +51,11 @@ SSLCipherSuite RC4-SHA:HIGH:!ADH
 AddType application/hal+json .haljson
 SSLEngine   On
 
+Include {{ parameter_dict.get('httpd-cors-config-file') }}
+Header set Access-Control-Allow-Credentials "true"
+Header set Access-Control-Allow-Methods "PROPFIND, PROPPATCH, COPY, MOVE, DELETE, MKCOL, LOCK, UNLOCK, PUT, GETLIB, VERSION-CONTROL, CHECKIN, CHECKOUT, UNCHECKOUT, REPORT, UPDATE, CANCELUPLOAD, HEAD, OPTIONS, GET, POST"
+Header set Access-Control-Allow-Headers "Overwrite, Destination, Content-Type, Depth, User-Agent, X-File-Size, X-Requested-With, If-Modified-Since, X-File-Name, Cache-Control, Authorization"
+
 {% if parameter_dict.has_key('monitor-url-list') -%}
 RewriteEngine on
 SSLProxyEngine on
@@ -66,6 +73,27 @@ RewriteRule /monitor{{ index }}/(.*)  {{ url }}/$1 [L,P]
 {% endfor -%}
 {% endif -%}
 
+DavLockDB {{ directory.get('monitor-var') }}/DavLock
+Alias /share {{ directory.get('webdav') }}
+<Directory {{ directory.get('webdav') }}>
+    DirectoryIndex disabled
+    DAV On
+    Options Indexes FollowSymLinks
+    AuthType Basic
+    AuthName "webdav"
+    AuthUserFile "{{ parameter_dict.get('htpasswd-file') }}"
+    <LimitExcept OPTIONS>
+        Require valid-user
+    </LimitExcept>
+</Directory>
+
+<LocationMatch "/share/(jio_)?public">
+    <Limit GET HEAD OPTIONS REPORT PROPFIND>
+        Allow from all
+        Satisfy any
+    </Limit>
+</LocationMatch>
+
 ScriptSock {{ parameter_dict.get('cgid-pid-file') }}
 <Directory {{ directory.get('www') }}>
   SSLVerifyDepth    1