Commit defc6519 authored by Alain Takoudjou's avatar Alain Takoudjou

Monitor: define public and private data

parent a3763e67
......@@ -92,7 +92,7 @@ md5sum = 04b664dfb47bfd3d01502768311aa239
# Monitor templates files
[monitor-httpd-conf]
<= monitor-template-base
md5sum = 68a7e4d3304fc78d173f30ad99534762
md5sum = 4db64f486ec7a6af9b5c0d9270ef52c9
filename = monitor-httpd.conf.in
[monitor-service-conf-template]
......@@ -113,7 +113,12 @@ md5sum = 2db5c08c7e8658981b4b1e3f27fd5967
[monitor-instance-info]
<= monitor-template-base
filename = instance-info.conf.in
md5sum = d506e335e366c1d873a3490333842398
md5sum = 1bdb4e05c6be04f4e5766c64467fbcec
[monitor-httpd-cors]
<= monitor-template-base
filename = httpd-cors.cfg.in
md5sum = 5afad2bb6e088e080e907f1d837effbb
# End templates files
......@@ -122,13 +127,14 @@ recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg
md5sum = 8763a92c2c22103a9f6c9052ba322fc5
md5sum = ee39ce85084d29172f50ae7375960371
context =
key apache_location apache:location
key gzip_location gzip:location
raw monitor_bin ${monitor2-bin:location}/${monitor2-bin:filename}
raw monitor_collect ${monitor-collect:location}/${monitor-collect:filename}
raw monitor_conf_template ${monitor-conf:location}/${monitor-conf:filename}
raw monitor_https_cors ${monitor-httpd-cors:location}/${monitor-httpd-cors:filename}
raw monitor_instance_info ${monitor-instance-info:location}/${monitor-instance-info:filename}
raw monitor_globalstate ${monitor-globalstate:location}/${monitor-globalstate:filename}
raw monitor_password_promise_template ${monitor-password-promise:location}/${monitor-password-promise:filename}
......@@ -158,13 +164,13 @@ context =
[monitor2-bin]
<= monitor-template-script
filename = monitor.py
md5sum = 3d2035d7c02b9d37c5c680e142dcb845
md5sum = 119827d7e7ed45c627f1ad21a94445b8
[run-promise-py]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/scripts/run-promise.py
rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
md5sum = 4f99c09125c201ee988e71662b8c9bf9
md5sum = 9a9dfa02742019cc86c78898822666bc
mode = 0755
context =
raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
......@@ -182,7 +188,7 @@ md5sum = 40dcda69f1ccb8639d7ef0ad65bceb5e
[monitor-globalstate]
<= monitor-template-script
filename = globalstate.py
md5sum = 906776302bfe511b21a7eee3c9bcb6a5
md5sum = 13f0cb7b87eb2ce91b4b27e2303ec01c
[monitor-collect]
<= monitor-template-script
......
# Put here default Name for instance and Hosting Subscription
[slap-configuration]
root-title = UNKNOW
title = UNKNOW Instance
[slap-configuration]
recipe = slapos.cookbook:slapconfiguration.serialised
computer = ${slap-connection:computer-id}
......@@ -74,7 +79,6 @@ services-conf = ${directory:etc}/monitor.conf.d
www = ${directory:monitor}/web
web-dir = ${directory:monitor}/web
log = ${directory:log}/monitor
promise-wrapper = ${directory:var}/monitor-promise-wrapper
monitor-var = ${directory:var}/monitor
monitor-password-var = ${monitor-directory:monitor-var}/password
monitor-password-interface = ${monitor-directory:monitor-password-var}/password/interface
......@@ -124,6 +128,7 @@ wrapper = ${directory:services}/monitor-httpd
[monitor-conf-parameters]
title = ${monitor-instance-parameter:monitor-title}
root-title = ${monitor-instance-parameter:root-instance-title}
public-folder = ${monitor-directory:public}
private-folder = ${monitor-directory:private}
webdav-folder = ${monitor-directory:webdav}
......@@ -133,7 +138,6 @@ monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
service-pid-folder = ${monitor-directory:pids}
crond-folder = ${logrotate-directory:cron-entries}
logrotate-folder = ${logrotate:logrotate-entries}
wraper-folder = ${monitor-directory:promise-wrapper}
promise-runner = {{ promise_executor_py }}
promise-folder-list =
${directory:promises}
......@@ -158,6 +162,7 @@ context = section parameter_dict monitor-conf-parameters
[instance-info-parameters]
name = ${monitor-instance-parameter:monitor-title}
root-name = ${monitor-instance-parameter:root-instance-title}
computer-id = ${slap-connection:computer-id}
ipv4 = ${slap-configuration:ipv4-random}
ipv6 = ${slap-configuration:ipv6-random}
......@@ -185,13 +190,18 @@ wrapper-path = ${directory:scripts}/bootstrap-monitor
environment =
PATH=${python-symlink:target}:/usr/local/bin:/usr/bin:/bin
[monitor-htpasswd]
recipe = slapos.cookbook:generate.password
storage-path = ${directory:etc}/.monitor_user
bytes = 8
[httpd-monitor-htpasswd]
recipe = plone.recipe.command
stop-on-error = true
htpasswd-path = ${monitor-directory:etc}/monitor-htpasswd
command = {{ apache_location }}/bin/htpasswd -cb ${:htpasswd-path} ${:user} ${:password}
user = admin
password = admin
password = ${monitor-htpasswd:passwd}
[monitor-httpd-conf-parameter]
listening-ip = ${monitor-instance-parameter:monitor-httpd-ipv6}
......@@ -204,7 +214,7 @@ cert-file = ${ca-directory:certs}/httpd.crt
key-file = ${ca-directory:certs}/httpd.key
htpasswd-file = ${httpd-monitor-htpasswd:htpasswd-path}
url = https://[${monitor-instance-parameter:monitor-httpd-ipv6}]:${:port}
httpd-cors-config-file = ${monitor-httpd-cors:location}
httpd-cors-config-file = ${monitor-httpd-cors:rendered}
[monitor-httpd-conf]
recipe = slapos.recipe.template:jinja2
......@@ -216,11 +226,12 @@ context =
section parameter_dict monitor-httpd-conf-parameter
[monitor-httpd-cors]
recipe = plone.recipe.command
command = if [ ! -f ${:location} ]; then echo 'Header set Access-Control-Allow-Origin "*"' > ${:location}; fi
location = ${directory:etc}/${:filename}
filename = httpd-cors.cfg
stop-on-error = true
recipe = slapos.recipe.template:jinja2
template = {{ monitor_https_cors }}
rendered = ${directory:etc}/httpd-cors.cfg
mode = 0600
context =
key domain monitor-instance-parameter:cors-domains
[httpd-wrapper]
recipe = slapos.cookbook:wrapper
......@@ -352,14 +363,38 @@ context =
[publish]
recipe = slapos.cookbook:publish
monitor-url = ${monitor-httpd-conf-parameter:url}
monitor-config-url = ${monitor-httpd-conf-parameter:url}/share/jio_public
#monitor-base-url = ${monitor-httpd-conf-parameter:url}
monitor-base-url = ${monitor-frontend-promise:url}
monitor-url = ${:monitor-base-url}/share/jio_public
monitor-user = ${httpd-monitor-htpasswd:user}
monitor-password = ${httpd-monitor-htpasswd:password}
[monitor-instance-parameter]
monitor-title = Monitoring interface
monitor-title = ${slap-configuration:title}
root-instance-title = ${slap-configuration:root-title}
cors-domains =
# XXX Hard coded parameter
collector-db = /srv/slapgrid/var/data-log/collector.db
[monitor-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = Monitor Frontend ${monitor-instance-parameter:monitor-title}
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
config-url = ${monitor-httpd-conf-parameter:url}
software-type = custom-personal
return = site_url domain
[monitor-frontend-promise]
recipe = slapos.cookbook:check_url_available
path = ${directory:promises}/monitor-http-frontend
url = ${monitor-frontend:connection-site_url}
dash_path = {{ dash_executable_location }}
curl_path = {{ curl_executable_location }}
check-secure = 1
[monitor-base]
# create dependencies between required monitor parts
recipe = plone.recipe.command
......
......@@ -6,6 +6,7 @@ import glob
import json
import ConfigParser
import time
from datetime import datetime
def softConfigGet(config, *args, **kwargs):
try:
......@@ -45,7 +46,8 @@ def main(args_list):
monitor_config = ConfigParser.ConfigParser()
monitor_config.read(monitor_file)
base_folder = monitor_config.get('monitor', 'public-folder')
base_folder = monitor_config.get('monitor', 'private-folder')
status_folder = monitor_config.get('monitor', 'public-folder')
base_url = monitor_config.get('monitor', 'base-url')
related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
statistic_folder = os.path.join(base_folder, 'data', '.jio_documents')
......@@ -60,13 +62,14 @@ def main(args_list):
# search for all status files
file_list = filter(os.path.isfile,
glob.glob("%s/*.status.json" % base_folder)
glob.glob("%s/*.status.json" % status_folder)
)
error = warning = success = 0
latest_date = ''
status = 'OK'
promise_list = []
global_state_file = os.path.join(base_folder, 'monitor.global.json')
public_state_file = os.path.join(status_folder, 'monitor.global.json')
for file in file_list:
try:
with open(file, 'r') as temp_file:
......@@ -100,9 +103,8 @@ def main(args_list):
},
date=latest_date,
_links={"rss_url": {"href": "%s/public/feed" % base_url},
"public_url": {"href": "%s/share/jio_public" % base_url},
"private_url": {"href": "%s/share/jio_private" % base_url},
"data_url": {"href": '%s/public/data' % base_url}
"public_url": {"href": "%s/share/jio_public/" % base_url},
"private_url": {"href": "%s/share/jio_private/" % base_url}
},
data={'state': 'monitor_state.data',
'process_state': 'monitor_process_resource.status',
......@@ -120,6 +122,7 @@ def main(args_list):
if 'instance' in config.sections():
instance_dict = {}
global_state_dict['title'] = config.get('instance', 'name')
global_state_dict['hosting-title'] = config.get('instance', 'root-name')
if not global_state_dict['title']:
global_state_dict['title'] = 'Instance Monitoring'
......@@ -136,9 +139,22 @@ def main(args_list):
global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/jio_public" % url}
for url in related_monitor_list]
# Public information with the link to private folder
public_state_dict = dict(
status=status,
date=latest_date,
_links={'monitor': {'href': '%s/share/jio_private/' % base_url}},
title=global_state_dict.get('title', '')
)
public_state_dict['hosting-title'] = global_state_dict.get('hosting-title', '')
public_state_dict['_links']['related_monitor'] = global_state_dict['_links'].get('related_monitor', [])
with open(global_state_file, 'w') as fglobal:
fglobal.write(json.dumps(global_state_dict))
with open(public_state_file, 'w') as fpglobal:
fpglobal.write(json.dumps(public_state_dict))
generateStatisticsData(
os.path.join(statistic_folder, 'monitor_state.data.json'),
global_state_dict)
......
......@@ -80,10 +80,10 @@ class Monitoring(object):
# Set Monitor variables
self.monitor_hal_json = config.get("monitor", "monitor-hal-json")
self.title = config.get("monitor", "title")
self.root_title = config.get("monitor", "root-title")
self.service_pid_folder = config.get("monitor", "service-pid-folder")
self.crond_folder = config.get("monitor", "crond-folder")
self.logrotate_d = config.get("monitor", "logrotate-folder")
self.wraper_folder = config.get("monitor", "wraper-folder")
self.promise_runner = config.get("monitor", "promise-runner")
self.promise_folder_list = config.get("monitor", "promise-folder-list").split()
self.public_folder = config.get("monitor", "public-folder")
......@@ -147,7 +147,7 @@ class Monitoring(object):
#configure jio_documents folder
jio_public = os.path.join(self.webdav_folder, 'jio_public')
jio_private = os.path.join(self.webdav_folder, 'jio_private', '.jio_documents')
jio_private = os.path.join(self.webdav_folder, 'jio_private')
mkdirAll(jio_public)
mkdirAll(jio_private)
mkdirAll(self.status_history_folder)
......@@ -162,6 +162,15 @@ class Monitoring(object):
if e.errno != os.errno.EEXIST:
raise
self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
mkdirAll(self.data_folder)
try:
os.symlink(os.path.join(self.private_folder, 'data'),
os.path.join(jio_private, 'data'))
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
def generateOpmlFile(self, feed_url_list, output_file):
if not feed_url_list:
return
......@@ -206,13 +215,13 @@ class Monitoring(object):
"public": {"href": "%s/public" % self.webdav_url},
"private": {"href": "%s/private" % self.webdav_url},
"rss": {"href": "%s/feed" % self.public_url},
"jio_public": {"href": "%s/jio_public" % self.webdav_url},
"jio_private": {"href": "%s/jio_private" % self.webdav_url}
"jio_public": {"href": "%s/jio_public/" % self.webdav_url},
"jio_private": {"href": "%s/jio_private/" % self.webdav_url}
}
if self.title:
self.monitor_dict["title"] = self.title
if self.monitor_url_list:
monitor_link_dict["related_monitor"] = [{"href": "%s/share/jio_public" % url}
monitor_link_dict["related_monitor"] = [{"href": url}
for url in self.monitor_url_list]
self.generateOpmlFile(self.monitor_url_list,
os.path.join(self.public_folder, 'feeds'))
......@@ -245,28 +254,29 @@ class Monitoring(object):
service_config = promise["configuration"]
service_status_path = "%s/%s.status.json" % (self.public_folder, service_name) # hardcoded
mkdirAll(os.path.dirname(service_status_path))
command = "%s %s %s %s %s %s %s" % (
self.promise_runner,
os.path.join(self.service_pid_folder, "%s.pid" % service_name),
service_status_path,
promise["path"],
service_name,
"%s/jio_public" % self.webdav_url, # XXX hardcoded
self.status_history_folder
)
cron_line_list.append("%s %s" % (
promise_cmd_line = [
softConfigGet(service_config, "service", "frequency") or "* * * * *",
command.replace("%", "\\%"),
))
self.promise_runner,
'--pid_path %s' % os.path.join(self.service_pid_folder,
"%s.pid" % service_name),
'--output %s' % service_status_path,
'--promise_script %s' % promise["path"],
'--promise_name "%s"' % service_name,
'--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder %s' % self.status_history_folder,
'--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title]
cron_line_list.append(' '.join(promise_cmd_line))
if service_name in service_name_list:
service_name_list.pop(service_name_list.index(service_name))
wrapper_path = os.path.join(self.wraper_folder, service_name)
"""wrapper_path = os.path.join(self.wraper_folder, service_name)
with open(wrapper_path, "w") as fp:
fp.write("#!/bin/sh\n%s" % command) # XXX hardcoded, use dash, sh or bash binary!
os.chmod(wrapper_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH )
os.chmod(wrapper_path, stat.S_IRUSR | stat.S_IWUSR | stat.S_IXUSR | stat.S_IRGRP | stat.S_IROTH )"""
if service_name_list != []:
# XXX Some service was removed, delete his status file so monitor will not consider his status anymore
......@@ -318,8 +328,6 @@ class Monitoring(object):
self.generateServiceCronEntries()
# Rotate monitor data files
self.data_folder = os.path.join(self.public_folder, 'data', '.jio_documents')
mkdirAll(self.data_folder)
option_list = [
'daily', 'nocreate', 'noolddir', 'rotate 30',
'nocompress', 'extension .json', 'dateext',
......
......@@ -9,22 +9,39 @@ import psutil
import time
from shutil import copyfile
import glob
import argparse
def parseArguments():
"""
Parse arguments for monitor collector instance.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--pid_path',
help='Path where the pid of this process will be writen.')
parser.add_argument('--output',
help='The Path of file where Json result of this promise will be saved.')
parser.add_argument('--promise_script',
help='Promise script to execute.')
parser.add_argument('--promise_name',
help='Title to give to this promise.')
parser.add_argument('--monitor_url',
help='Monitor Instance website URL.')
parser.add_argument('--history_folder',
help='Path where old result file will be placed before generate a new json result file.')
parser.add_argument('--instance_name',
default='UNKNOW Software Instance',
help='Software Instance name.')
parser.add_argument('--hosting_name',
default='UNKNOW Hosting Subscription',
help='Hosting Subscription name.')
return parser.parse_args()
def main():
if len(sys.argv) < 4:
print("Usage: %s <pid_path> <output_path> <command> [<name>] [...]" % sys.argv[0])
return 2
pid_path=sys.argv[1]
output_path=sys.argv[2]
promise_name = history_folder = related_url = ""
if len(sys.argv) >= 5:
promise_name = sys.argv[4]
if len(sys.argv) >= 6:
related_url = sys.argv[5]
if len(sys.argv) >= 7:
history_folder = sys.argv[6]
if os.path.exists(pid_path):
with open(pid_path, "r") as pidfile:
parser = parseArguments()
if os.path.exists(parser.pid_path):
with open(parser.pid_path, "r") as pidfile:
try:
pid = int(pidfile.read(6))
except ValueError:
......@@ -33,29 +50,35 @@ def main():
print("A process is already running with pid " + str(pid))
return 1
start_date = ""
with open(pid_path, "w") as pidfile:
process = executeCommand(sys.argv[3:])
with open(parser.pid_path, "w") as pidfile:
process = executeCommand(parser.promise_script)
ps_process = psutil.Process(process.pid)
start_date = time.strftime("%Y-%m-%d %H:%M:%S", time.localtime(ps_process.create_time()))
pidfile.write(str(process.pid))
status_json = generateStatusJsonFromProcess(process, start_date=start_date, title=promise_name)
status_json = generateStatusJsonFromProcess(process, start_date=start_date)
status_json['_links'] = {"monitor": {"href": parser.monitor_url}}
status_json['title'] = parser.promise_name
status_json['instance'] = parser.instance_name
status_json['hosting_subscription'] = parser.hosting_name
# Save the lastest status change date (needed for rss)
if related_url:
status_json['_links'] = {"monitor": {"href": related_url}}
status_json['change-time'] = ps_process.create_time()
if os.path.exists(output_path):
with open(output_path) as f:
if os.path.exists(parser.output):
with open(parser.output) as f:
last_result = json.loads(f.read())
if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
status_json['change-time'] = last_result['change-time']
if history_folder:
updateStatusHistoryFolder(promise_name, output_path, history_folder)
with open(output_path, "w") as outputfile:
updateStatusHistoryFolder(
parser.promise_name,
parser.output,
parser.history_folder
)
with open(parser.output, "w") as outputfile:
json.dump(status_json, outputfile)
os.remove(pid_path)
os.remove(parser.pid_path)
def updateStatusHistoryFolder(name, status_file, history_folder):
old_history_list = []
......
{% if domain -%}
{% set allow_domain = '|'.join(domain.replace('.', '\.').split()) -%}
SetEnvIf Origin "^http(s)?://(.+\.)?({{ allow_domain }})$" ORIGIN_DOMAIN=$0
Header always set Access-Control-Allow-Origin "%{ORIGIN_DOMAIN}e" env=ORIGIN_DOMAIN
{% endif -%}
\ No newline at end of file
[instance]
name = {{ instance_dict['name'] }}
root-name = {{ instance_dict['root-name'] }}
computer = {{ instance_dict['computer-id'] }}
ipv4 = {{ instance_dict['ipv4'] }}
ipv6 = {{ instance_dict['ipv6'] }}
......
......@@ -33,6 +33,8 @@ LoadModule rewrite_module modules/mod_rewrite.so
LoadModule headers_module modules/mod_headers.so
LoadModule dav_module modules/mod_dav.so
LoadModule dav_fs_module modules/mod_dav_fs.so
LoadModule env_module modules/mod_env.so
LoadModule setenvif_module modules/mod_setenvif.so
# SSL Configuration
<IfDefine !SSLConfigured>
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment