Monitor: define public and private data

stack/monitor2/buildout.cfg

@@ -92,7 +92,7 @@ md5sum = 04b664dfb47bfd3d01502768311aa239
 # Monitor templates files
 [monitor-httpd-conf]
 <= monitor-template-base
-md5sum = 68a7e4d3304fc78d173f30ad99534762
+md5sum = 4db64f486ec7a6af9b5c0d9270ef52c9
 filename = monitor-httpd.conf.in
 
 [monitor-service-conf-template]
@@ -113,7 +113,12 @@ md5sum = 2db5c08c7e8658981b4b1e3f27fd5967
 [monitor-instance-info]
 <= monitor-template-base
 filename = instance-info.conf.in
-md5sum = d506e335e366c1d873a3490333842398
+md5sum = 1bdb4e05c6be04f4e5766c64467fbcec
+
+[monitor-httpd-cors]
+<= monitor-template-base
+filename = httpd-cors.cfg.in
+md5sum = 5afad2bb6e088e080e907f1d837effbb
 # End templates files
 
 
@@ -122,13 +127,14 @@ recipe = slapos.recipe.template:jinja2
 filename = template-monitor.cfg
 template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
 rendered = ${buildout:directory}/template-monitor.cfg
-md5sum = 8763a92c2c22103a9f6c9052ba322fc5
+md5sum = ee39ce85084d29172f50ae7375960371
 context =
     key apache_location apache:location
     key gzip_location gzip:location
     raw monitor_bin ${monitor2-bin:location}/${monitor2-bin:filename}
     raw monitor_collect ${monitor-collect:location}/${monitor-collect:filename}
     raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
+    raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
     raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
     raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename}
     raw monitor_password_promise_template ${monitor-password-promise:location}/${monitor-password-promise:filename}
@@ -158,13 +164,13 @@ context =
 [monitor2-bin]
 <= monitor-template-script
 filename = monitor.py
-md5sum = 3d2035d7c02b9d37c5c680e142dcb845
+md5sum = 119827d7e7ed45c627f1ad21a94445b8
 
 [run-promise-py]
 recipe = slapos.recipe.template:jinja2
 template = ${:_profile_base_location_}/scripts/run-promise.py
 rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
-md5sum = 4f99c09125c201ee988e71662b8c9bf9
+md5sum = 9a9dfa02742019cc86c78898822666bc
 mode = 0755
 context =
   raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
@@ -182,7 +188,7 @@ md5sum = 40dcda69f1ccb8639d7ef0ad65bceb5e
 [monitor-globalstate]
 <= monitor-template-script
 filename = globalstate.py
-md5sum = 906776302bfe511b21a7eee3c9bcb6a5
+md5sum = 13f0cb7b87eb2ce91b4b27e2303ec01c
 
 [monitor-collect]
 <= monitor-template-script

stack/monitor2/instance-monitor.cfg.jinja2.in

+# Put here default Name for instance and Hosting Subscription
+[slap-configuration]
+root-title = UNKNOW
+title = UNKNOW Instance
+
 [slap-configuration]
 recipe = slapos.cookbook:slapconfiguration.serialised
 computer = ${slap-connection:computer-id}
@@ -74,7 +79,6 @@ services-conf = ${directory:etc}/monitor.conf.d
 www = ${directory:monitor}/web
 web-dir = ${directory:monitor}/web
 log = ${directory:log}/monitor
-promise-wrapper = ${directory:var}/monitor-promise-wrapper
 monitor-var = ${directory:var}/monitor
 monitor-password-var = ${monitor-directory:monitor-var}/password
 monitor-password-interface = ${monitor-directory:monitor-password-var}/password/interface
@@ -124,6 +128,7 @@ wrapper = ${directory:services}/monitor-httpd
 
 [monitor-conf-parameters]
 title = ${monitor-instance-parameter:monitor-title}
+root-title = ${monitor-instance-parameter:root-instance-title}
 public-folder = ${monitor-directory:public}
 private-folder = ${monitor-directory:private}
 webdav-folder = ${monitor-directory:webdav}
@@ -133,7 +138,6 @@ monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
 service-pid-folder = ${monitor-directory:pids}
 crond-folder = ${logrotate-directory:cron-entries}
 logrotate-folder = ${logrotate:logrotate-entries}
-wraper-folder = ${monitor-directory:promise-wrapper}
 promise-runner = {{ promise_executor_py }}
 promise-folder-list = 
   ${directory:promises}
@@ -158,6 +162,7 @@ context = section parameter_dict monitor-conf-parameters
 
 [instance-info-parameters]
 name = ${monitor-instance-parameter:monitor-title}
+root-name = ${monitor-instance-parameter:root-instance-title}
 computer-id = ${slap-connection:computer-id}
 ipv4 = ${slap-configuration:ipv4-random}
 ipv6 = ${slap-configuration:ipv6-random}
@@ -185,13 +190,18 @@ wrapper-path = ${directory:scripts}/bootstrap-monitor
 environment = 
   PATH=${python-symlink:target}:/usr/local/bin:/usr/bin:/bin
 
+[monitor-htpasswd]
+recipe = slapos.cookbook:generate.password
+storage-path = ${directory:etc}/.monitor_user
+bytes = 8
+
 [httpd-monitor-htpasswd]
 recipe = plone.recipe.command
 stop-on-error = true
 htpasswd-path = ${monitor-directory:etc}/monitor-htpasswd
 command = {{ apache_location }}/bin/htpasswd -cb ${:htpasswd-path} ${:user} ${:password}
 user = admin
-password = admin
+password = ${monitor-htpasswd:passwd}
 
 [monitor-httpd-conf-parameter]
 listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6}
@@ -204,7 +214,7 @@ cert-file = ${ca-directory:certs}/httpd.crt
 key-file = ${ca-directory:certs}/httpd.key
 htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path}
 url = https://[${monitor-instance-parameter:monitor-httpd-ipv6}]:${:port}
-httpd-cors-config-file = ${monitor-httpd-cors:location}
+httpd-cors-config-file = ${monitor-httpd-cors:rendered}
 
 [monitor-httpd-conf]
 recipe = slapos.recipe.template:jinja2
@@ -216,11 +226,12 @@ context =
   section parameter_dict monitor-httpd-conf-parameter
 
 [monitor-httpd-cors]
-recipe = plone.recipe.command
-command = if [ ! -f ${:location} ]; then echo 'Header set Access-Control-Allow-Origin "*"' > ${:location}; fi
-location = ${directory:etc}/${:filename}
-filename = httpd-cors.cfg
-stop-on-error = true
+recipe = slapos.recipe.template:jinja2
+template = {{ monitor_https_cors }}
+rendered = ${directory:etc}/httpd-cors.cfg
+mode = 0600
+context =
+  key domain monitor-instance-parameter:cors-domains
 
 [httpd-wrapper]
 recipe = slapos.cookbook:wrapper
@@ -352,14 +363,38 @@ context =
 
 [publish]
 recipe = slapos.cookbook:publish
-monitor-url = ${monitor-httpd-conf-parameter:url}
-monitor-config-url = ${monitor-httpd-conf-parameter:url}/share/jio_public
+#monitor-base-url = ${monitor-httpd-conf-parameter:url}
+monitor-base-url = ${monitor-frontend-promise:url}
+monitor-url = ${:monitor-base-url}/share/jio_public
+monitor-user = ${httpd-monitor-htpasswd:user}
+monitor-password = ${httpd-monitor-htpasswd:password}
 
 [monitor-instance-parameter]
-monitor-title = Monitoring interface
+monitor-title = ${slap-configuration:title}
+root-instance-title = ${slap-configuration:root-title}
+cors-domains = 
 # XXX Hard coded parameter
 collector-db = /srv/slapgrid/var/data-log/collector.db
 
+[monitor-frontend]
+<= slap-connection
+recipe = slapos.cookbook:requestoptional
+name = Monitor Frontend ${monitor-instance-parameter:monitor-title}
+# XXX We have hardcoded SR URL here.
+software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
+slave = true
+config-url = ${monitor-httpd-conf-parameter:url}
+software-type = custom-personal
+return = site_url domain
+
+[monitor-frontend-promise]
+recipe = slapos.cookbook:check_url_available
+path = ${directory:promises}/monitor-http-frontend
+url = ${monitor-frontend:connection-site_url}
+dash_path = {{ dash_executable_location }}
+curl_path = {{ curl_executable_location }}
+check-secure = 1
+
 [monitor-base]
 # create dependencies between required monitor parts
 recipe = plone.recipe.command

stack/monitor2/scripts/globalstate.py

@@ -6,6 +6,7 @@ import glob
 import json
 import ConfigParser
 import time
+from datetime import datetime
 
 def softConfigGet(config, *args, **kwargs):
   try:
@@ -45,7 +46,8 @@ def main(args_list):
   monitor_config = ConfigParser.ConfigParser()
   monitor_config.read(monitor_file)
 
-  base_folder = monitor_config.get('monitor', 'public-folder')
+  base_folder = monitor_config.get('monitor', 'private-folder')
+  status_folder = monitor_config.get('monitor', 'public-folder')
   base_url = monitor_config.get('monitor', 'base-url')
   related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
   statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
@@ -60,13 +62,14 @@ def main(args_list):
 
   # search for all status files
   file_list = filter(os.path.isfile,
-      glob.glob("%s/*.status.json" % base_folder)
+      glob.glob("%s/*.status.json" % status_folder)
     )
   error = warning = success = 0
   latest_date = ''
   status = 'OK'
   promise_list = []
   global_state_file = os.path.join(base_folder, 'monitor.global.json')
+  public_state_file = os.path.join(status_folder, 'monitor.global.json')
   for file in file_list:
     try:
       with open(file, 'r') as temp_file:
@@ -100,9 +103,8 @@ def main(args_list):
       },
       date=latest_date,
       _links={"rss_url": {"href": "%s/public/feed" % base_url},
-              "public_url": {"href": "%s/share/jio_public" % base_url},
-              "private_url": {"href": "%s/share/jio_private" % base_url},
-              "data_url": {"href": '%s/public/data' % base_url}
+              "public_url": {"href": "%s/share/jio_public/" % base_url},
+              "private_url": {"href": "%s/share/jio_private/" % base_url}
             },
       data={'state': 'monitor_state.data',
             'process_state': 'monitor_process_resource.status',
@@ -120,6 +122,7 @@ def main(args_list):
     if 'instance' in config.sections():
       instance_dict = {}
       global_state_dict['title'] = config.get('instance', 'name')
+      global_state_dict['hosting-title'] = config.get('instance', 'root-name')
       if not global_state_dict['title']:
         global_state_dict['title'] = 'Instance Monitoring'
       
@@ -136,9 +139,22 @@ def main(args_list):
     global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/jio_public" % url}
                           for url in related_monitor_list]
 
+  # Public information with the link to private folder
+  public_state_dict = dict(
+    status=status,
+    date=latest_date,
+    _links={'monitor': {'href': '%s/share/jio_private/' % base_url}},
+    title=global_state_dict.get('title', '')
+  )
+  public_state_dict['hosting-title'] = global_state_dict.get('hosting-title', '')
+  public_state_dict['_links']['related_monitor'] = global_state_dict['_links'].get('related_monitor', [])
+
   with open(global_state_file, 'w') as fglobal:
     fglobal.write(json.dumps(global_state_dict))
 
+  with open(public_state_file, 'w') as fpglobal:
+    fpglobal.write(json.dumps(public_state_dict))
+
   generateStatisticsData(
     os.path.join(statistic_folder, 'monitor_state.data.json'),
     global_state_dict)

stack/monitor2/scripts/monitor.py

@@ -80,10 +80,10 @@ class Monitoring(object):
     # Set Monitor variables
     self.monitor_hal_json = config.get("monitor", "monitor-hal-json")
     self.title = config.get("monitor", "title")
+    self.root_title = config.get("monitor", "root-title")
     self.service_pid_folder = config.get("monitor", "service-pid-folder")
     self.crond_folder = config.get("monitor", "crond-folder")
     self.logrotate_d = config.get("monitor", "logrotate-folder")
-    self.wraper_folder = config.get("monitor", "wraper-folder")
     self.promise_runner = config.get("monitor", "promise-runner")
     self.promise_folder_list = config.get("monitor", "promise-folder-list").split()
     self.public_folder = config.get("monitor", "public-folder")
@@ -147,7 +147,7 @@ class Monitoring(object):
 
     #configure jio_documents folder
     jio_public = os.path.join(self.webdav_folder, 'jio_public')
-    jio_private = os.path.join(self.webdav_folder, 'jio_private', '.jio_documents')
+    jio_private = os.path.join(self.webdav_folder, 'jio_private')
     mkdirAll(jio_public)
     mkdirAll(jio_private)
     mkdirAll(self.status_history_folder)
@@ -162,6 +162,15 @@ class Monitoring(object):
       if e.errno != os.errno.EEXIST:
         raise
 
+    self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
+    mkdirAll(self.data_folder)
+    try:
+      os.symlink(os.path.join(self.private_folder, 'data'),
+                  os.path.join(jio_private, 'data'))
+    except OSError, e:
+      if e.errno != os.errno.EEXIST:
+        raise
+
   def generateOpmlFile(self, feed_url_list, output_file):
     if not feed_url_list:
       return
@@ -206,13 +215,13 @@ class Monitoring(object):
                           "public": {"href": "%s/public" % self.webdav_url},
                           "private": {"href": "%s/private" % self.webdav_url},
                           "rss": {"href": "%s/feed" % self.public_url},
-                          "jio_public": {"href": "%s/jio_public" % self.webdav_url},
-                          "jio_private": {"href": "%s/jio_private" % self.webdav_url}
+                          "jio_public": {"href": "%s/jio_public/" % self.webdav_url},
+                          "jio_private": {"href": "%s/jio_private/" % self.webdav_url}
                         }
     if self.title:
       self.monitor_dict["title"] = self.title
     if self.monitor_url_list:
-      monitor_link_dict["related_monitor"] = [{"href": "%s/share/jio_public" % url}
+      monitor_link_dict["related_monitor"] = [{"href": url}
                                   for url in self.monitor_url_list]
       self.generateOpmlFile(self.monitor_url_list,
         os.path.join(self.public_folder, 'feeds'))
@@ -245,28 +254,29 @@ class Monitoring(object):
       service_config = promise["configuration"]
       service_status_path = "%s/%s.status.json" % (self.public_folder, service_name)  # hardcoded
       mkdirAll(os.path.dirname(service_status_path))
-      command = "%s %s %s %s %s %s %s" % (
-          self.promise_runner,
-          os.path.join(self.service_pid_folder, "%s.pid" % service_name),
-          service_status_path,
-          promise["path"],
-          service_name,
-          "%s/jio_public" % self.webdav_url, # XXX hardcoded
-          self.status_history_folder
-        )
-
-      cron_line_list.append("%s %s" % (
-          softConfigGet(service_config, "service", "frequency") or "* * * * *",
-          command.replace("%", "\\%"),
-        ))
+
+      promise_cmd_line = [
+        softConfigGet(service_config, "service", "frequency") or "* * * * *",
+        self.promise_runner,
+        '--pid_path %s' % os.path.join(self.service_pid_folder,
+          "%s.pid" % service_name),
+        '--output %s' % service_status_path,
+        '--promise_script %s' % promise["path"],
+        '--promise_name "%s"' % service_name,
+        '--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
+        '--history_folder %s' % self.status_history_folder,
+        '--instance_name "%s"' % self.title,
+        '--hosting_name "%s"' % self.root_title]
+
+      cron_line_list.append(' '.join(promise_cmd_line))
 
       if service_name in service_name_list:
         service_name_list.pop(service_name_list.index(service_name))
 
-      wrapper_path = os.path.join(self.wraper_folder, service_name)
+      """wrapper_path = os.path.join(self.wraper_folder, service_name)
       with open(wrapper_path, "w") as fp:
         fp.write("#!/bin/sh\n%s" % command)  # XXX hardcoded, use dash, sh or bash binary!
-      os.chmod(wrapper_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH )
+      os.chmod(wrapper_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH )"""
 
     if service_name_list != []:
       # XXX Some service was removed, delete his status file so monitor will not consider his status anymore
@@ -318,8 +328,6 @@ class Monitoring(object):
     self.generateServiceCronEntries()
 
     # Rotate monitor data files
-    self.data_folder = os.path.join(self.public_folder, 'data', '.jio_documents')
-    mkdirAll(self.data_folder)
     option_list = [
       'daily', 'nocreate', 'noolddir', 'rotate 30',
       'nocompress', 'extension .json', 'dateext',

stack/monitor2/scripts/run-promise.py

@@ -9,22 +9,39 @@ import psutil
 import time
 from shutil import copyfile
 import glob
+import argparse
+
+def parseArguments():
+  """
+  Parse arguments for monitor collector instance.
+  """
+  parser = argparse.ArgumentParser()
+  parser.add_argument('--pid_path',
+                      help='Path where the pid of this process will be writen.')
+  parser.add_argument('--output',
+                      help='The Path of file where Json result of this promise will be saved.')
+  parser.add_argument('--promise_script',
+                      help='Promise script to execute.')
+  parser.add_argument('--promise_name',
+                      help='Title to give to this promise.')
+  parser.add_argument('--monitor_url',
+                      help='Monitor Instance website URL.')
+  parser.add_argument('--history_folder',
+                      help='Path where old result file will be placed before generate a new json result file.')
+  parser.add_argument('--instance_name',
+                      default='UNKNOW Software Instance',
+                      help='Software Instance name.')
+  parser.add_argument('--hosting_name',
+                      default='UNKNOW Hosting Subscription',
+                      help='Hosting Subscription name.')
+
+  return parser.parse_args()
 
 def main():
-  if len(sys.argv) < 4:
-    print("Usage: %s <pid_path> <output_path> <command> [<name>] [...]" % sys.argv[0])
-    return 2
-  pid_path=sys.argv[1]
-  output_path=sys.argv[2]
-  promise_name = history_folder = related_url = ""
-  if len(sys.argv) >= 5:
-    promise_name = sys.argv[4]
-  if len(sys.argv) >= 6:
-    related_url = sys.argv[5]
-  if len(sys.argv) >= 7:
-    history_folder = sys.argv[6]
-  if os.path.exists(pid_path):
-    with open(pid_path, "r") as pidfile:
+  parser = parseArguments()
+
+  if os.path.exists(parser.pid_path):
+    with open(parser.pid_path, "r") as pidfile:
       try:
         pid = int(pidfile.read(6))
       except ValueError:
@@ -33,29 +50,35 @@ def main():
         print("A process is already running with pid " + str(pid))
         return 1
   start_date = ""
-  with open(pid_path, "w") as pidfile:
-    process = executeCommand(sys.argv[3:])
+  with open(parser.pid_path, "w") as pidfile:
+    process = executeCommand(parser.promise_script)
     ps_process = psutil.Process(process.pid)
     start_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ps_process.create_time()))
     pidfile.write(str(process.pid))
-  
-  status_json = generateStatusJsonFromProcess(process, start_date=start_date, title=promise_name)
+
+  status_json = generateStatusJsonFromProcess(process, start_date=start_date)
+
+  status_json['_links'] = {"monitor": {"href": parser.monitor_url}}
+  status_json['title'] = parser.promise_name
+  status_json['instance'] = parser.instance_name
+  status_json['hosting_subscription'] = parser.hosting_name
 
   # Save the lastest status change date (needed for rss)
-  if related_url:
-    status_json['_links'] = {"monitor": {"href": related_url}}
   status_json['change-time'] = ps_process.create_time()
-  if os.path.exists(output_path):
-    with open(output_path) as f:
+  if os.path.exists(parser.output):
+    with open(parser.output) as f:
       last_result = json.loads(f.read())
       if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
         status_json['change-time'] = last_result['change-time']
 
-  if history_folder:
-    updateStatusHistoryFolder(promise_name, output_path, history_folder)
-  with open(output_path, "w") as outputfile:
+  updateStatusHistoryFolder(
+    parser.promise_name,
+    parser.output,
+    parser.history_folder
+  )
+  with open(parser.output, "w") as outputfile:
     json.dump(status_json, outputfile)
-  os.remove(pid_path)
+  os.remove(parser.pid_path)
 
 def updateStatusHistoryFolder(name, status_file, history_folder):
   old_history_list = []

stack/monitor2/templates/httpd-cors.cfg.in

+{% if domain -%}
+{% set allow_domain = '|'.join(domain.replace('.', '\.').split()) -%}
+SetEnvIf Origin "^http(s)?://(.+\.)?({{ allow_domain }})$" ORIGIN_DOMAIN=$0
+Header always set Access-Control-Allow-Origin "%{ORIGIN_DOMAIN}e" env=ORIGIN_DOMAIN
+{% endif -%}
\ No newline at end of file

stack/monitor2/templates/instance-info.conf.in

 [instance]
 name = {{ instance_dict['name'] }}
+root-name = {{ instance_dict['root-name'] }}
 computer = {{ instance_dict['computer-id'] }}
 ipv4 = {{ instance_dict['ipv4'] }}
 ipv6 = {{ instance_dict['ipv6'] }}

stack/monitor2/templates/monitor-httpd.conf.in

@@ -33,6 +33,8 @@ LoadModule rewrite_module modules/mod_rewrite.so
 LoadModule headers_module modules/mod_headers.so
 LoadModule dav_module modules/mod_dav.so
 LoadModule dav_fs_module modules/mod_dav_fs.so
+LoadModule env_module modules/mod_env.so
+LoadModule setenvif_module modules/mod_setenvif.so
 
 # SSL Configuration
 <IfDefine !SSLConfigured>