Commit e54a96b4 authored by Alain Takoudjou's avatar Alain Takoudjou

Merge branch 'master' into 1.0

parents e8faba06 98a6cd5c
......@@ -155,16 +155,11 @@ class GenericBaseRecipe(object):
lines.append(dedent("""
# Check for other instances
pidfile=%s
if [ -e $pidfile ]; then
pid=$(cat $pidfile)
if [ ! -z "$(ps -p $pid | grep $(basename %s))" ]; then
if pid=`pgrep -F $pidfile -f "$COMMAND" 2>/dev/null`; then
echo "Already running with pid $pid."
exit 1
else
rm $pidfile
fi
fi
echo $$ > $pidfile""" % (shlex.quote(pidfile), command)))
echo $$ > $pidfile""" % shlex.quote(pidfile)))
lines.append(dedent('''
# If the wrapped command uses a shebang, execute the referenced
......
......@@ -54,7 +54,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner.cfg
output = ${buildout:directory}/template-runner.cfg.in
md5sum = 8fedcc83e2ecb488910ddd8a1c82fe24
md5sum = 66dba3331f49492d317e9c14d68d1bcb
mode = 0644
[template-runner-import-script]
......@@ -69,7 +69,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-import.cfg.in
output = ${buildout:directory}/instance-runner-import.cfg
md5sum = 8ae80f9a9d5523219e1c9065f1cab6d8
md5sum = 673c30e5e7f9b7bb543f79465a56e43d
mode = 0644
[template-runner-export-script]
......@@ -84,13 +84,13 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-runner-export.cfg.in
output = ${buildout:directory}/instance-runner-export.cfg
md5sum = 8f4912ca04a650298c3c260689109c2e
md5sum = 4b0ab39bc655ae0b865207147cb2e5bf
mode = 0644
[template-resilient]
recipe = slapos.recipe.build:download
url = ${:_profile_base_location_}/instance-resilient.cfg.jinja2
md5sum = 160bf71d8a15f1359d80eb5586109781
md5sum = 6373b2cf3c0cee78bd858959976271d3
filename = instance-resilient.cfg.jinja2
mode = 0644
......
......@@ -43,11 +43,6 @@ username = admin
recipe = slapos.cookbook:mkdirectory
etc = ${buildout:directory}/etc
# XXX Monitoring Main Instane
[monitor-instance-parameter]
monitor-httpd-port = 8160
cors-domains = {{ monitor_parameter.get('monitor-cors-domains', '') }}
# Bubble up the parameters
[request-runner]
......
......@@ -35,6 +35,7 @@ parts +=
httpd-graceful-wrapper
## Monitoring part
## Monitor for runner
monitor-base
monitor-check-resilient-feed-file
monitor-check-webrunner-internal-instance
......@@ -66,7 +67,7 @@ instance-configuration =
[monitor-check-resilient-feed-file]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
rendered = $${monitor-directory:promises}/check-create-resilient-feed-files
rendered = $${monitor-directory:reports}/check-create-resilient-feed-files
mode = 700
context =
key input_feed_directory directory:notifier-feeds
......
......@@ -29,6 +29,8 @@ parts +=
# have to repeat the next one, as it's not inherited from pbsready-import
import-on-notification
monitor-base
# For the needs of importer, we run the full slaprunner
# In case both exporter and importer (aka main instance and clone instance)
# run with the same IP (usually for testing purposes),
......
......@@ -34,7 +34,7 @@ parts =
custom-frontend-promise
{% endif %}
## Monitoring part
monitor-base
monitor-check-webrunner-internal-instance
extends = ${monitor2-template:rendered}
......@@ -308,9 +308,7 @@ path_access_log = $${directory:log}/httpd-access.log
cert_file = $${ca-nginx:cert-file}
key_file = $${ca-nginx:key-file}
global_ip = $${slap-network-information:global-ipv6}
global_port = 8386
#httpd_port = $${monitor-parameters:port}
#monitor_index = $${deploy-index:rendered}
global_port = $${slap-parameter:slaprunner-httpd-port}
working_directory = $${slaprunner:working-directory}
dav_lock = $${directory:var}/WebDavLock
htpasswd_file = $${monitor-httpd-conf-parameter:htpasswd-file}
......@@ -441,7 +439,9 @@ return = site_url domain
[request-httpd-frontend]
<= slap-connection
recipe = slapos.cookbook:requestoptional
name = SlapRunner httpd Frontend
# XXX - Unfortunately, we still call webrunner httpd frontend "Monitor Frontend" otherwise
# buildout will ignore previous frontend that was created and create a new one (in case of upgrade)
name = Monitor Frontend
# XXX We have hardcoded SR URL here.
software-url = http://git.erp5.org/gitweb/slapos.git/blob_plain/HEAD:/software/apache-frontend/software.cfg
slave = true
......@@ -469,7 +469,7 @@ backend_url = $${slaprunner:access-url}
access_url = $${:url}/login
url = https://$${request-frontend:connection-domain}
ssh_command = ssh $${dropbear-runner-server:host} -p $${dropbear-runner-server:port}
webdav_url = $${request-httpd-frontend:connection-secure_access}/shared/
webdav_url = $${request-httpd-frontend:connection-secure_access}/share/
public_url = $${request-httpd-frontend:connection-secure_access}/public/
git_public_url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git-public/
git_private_url = https://[$${httpd-parameters:global_ip}]:$${httpd-parameters:global_port}/git/
......@@ -524,10 +524,14 @@ slapos-reference = master
auto-deploy = false
auto-deploy-instance = true
autorun = false
monitor-port = 9687
slaprunner-httpd-port = $${:monitor-port}
# XXX - for backward compatibility, monitor-port was for slaprunner httpd server
monitor-port = 9684
instance-name =
monitor-cors-domains =
monitor-interface-url =
# XXX - define a new port for monitor here and use monitor-port for backward compatibility
monitor-httpd-port = 8386
[monitor-parameters]
port = $${slap-parameter:monitor-port}
......@@ -718,7 +722,7 @@ port = $${supervisord:port}
# XXX Monitor
[monitor-instance-parameter]
monitor-httpd-port = $${slap-parameter:monitor-port}
monitor-httpd-port = $${slap-parameter:monitor-httpd-port}
{% if slapparameter_dict.get('name', '') -%}
monitor-title = {{ slapparameter_dict['name'] }}
{% endif -%}
......@@ -732,8 +736,8 @@ password = {{ slapparameter_dict['monitor-password'] }}
{% if slapparameter_dict.get('monitor-url-list', '') -%}
opml-url-list = {{ slapparameter_dict['monitor-url-list'] }}
{% endif -%}
# check if not resilient runner
{% if not slapparameter_dict.get('authorized-key', '') -%}
# Parameter for resilient
# Pass some parameter to dispay in monitoring interface
instance-configuration =
file recovery-code $${recovery-code:storage-path}
......@@ -745,7 +749,7 @@ configuration-file-path = $${buildout:directory}/knowledge0.cfg
[monitor-check-webrunner-internal-instance]
recipe = slapos.recipe.template:jinja2
template = ${monitor-check-webrunner-internal-instance:location}/${monitor-check-webrunner-internal-instance:filename}
rendered = $${monitor-directory:promises}/$${:filename}
rendered = $${monitor-directory:reports}/$${:filename}
filename = monitor-check-webrunner-internal-instance
mode = 0744
......@@ -142,5 +142,5 @@ setuptools = 0.9.8
prettytable = 0.7.3-nxd001
[networkcache]
download-cache-url = http://download.shacache.org/
download-dir-url = http://dir.shacache.org/
\ No newline at end of file
download-cache-url = http://www.shacache.org/shacache
download-dir-url = http://www.shacache.org/shadir
\ No newline at end of file
......@@ -99,7 +99,7 @@ recipe = slapos.recipe.template:jinja2
filename = template-monitor.cfg
template = ${:_profile_base_location_}/instance-monitor.cfg.jinja2.in
rendered = ${buildout:directory}/template-monitor.cfg
md5sum = cd7f386fe1b5066d8508758249b408d3
md5sum = b6da7709da533c5c8280e5e103809c58
context =
key apache_location apache:location
key gzip_location gzip:location
......@@ -130,13 +130,13 @@ depends =
[monitor2-bin]
<= monitor-template-script
filename = monitor.py
md5sum = 31beec15d3c3cd7979d04ecb834c439a
md5sum = 6db3d9fcfff32c556f2bedee804ffb47
[run-promise-py]
recipe = slapos.recipe.template:jinja2
template = ${:_profile_base_location_}/scripts/run-promise.py
rendered = ${buildout:parts-directory}/monitor-scripts/run-promise.py
md5sum = 8ba8b661c55f2c5a379e9e42573be486
md5sum = 5694920c5789f4d953cb5d41adfa7f8b
mode = 0755
context =
raw python ${buildout:directory}/bin/${extra-eggs:interpreter}
......
......@@ -70,7 +70,7 @@ etc = ${directory:etc}
run = ${directory:monitor}/run
#run = ${directory:scripts}
promises = ${directory:monitor-promise}
report = ${directory:monitor-report}
reports = ${directory:monitor-report}
pids = ${directory:run}/monitor
cgi-bin = ${directory:monitor}/cgi-bin
webdav = ${directory:monitor}/webdav
......@@ -134,6 +134,7 @@ root-title = ${monitor-instance-parameter:root-instance-title}
public-folder = ${monitor-directory:public}
private-folder = ${monitor-directory:private}
webdav-folder = ${monitor-directory:webdav}
report-folder = ${monitor-directory:reports}
web-folder = ${monitor-directory:web-dir}
base-url = ${monitor-instance-parameter:monitor-base-url}
monitor-hal-json = ${monitor-directory:public}/monitor.hal.json
......
......@@ -94,6 +94,7 @@ class Monitoring(object):
self.collector_db = config.get("monitor", "collector-db")
self.collect_script = config.get("monitor", "collect-script")
self.webdav_folder = config.get("monitor", "webdav-folder")
self.report_script_folder = config.get("monitor", "report-folder")
self.webdav_url = '%s/share' % config.get("monitor", "base-url")
self.public_url = '%s/public' % config.get("monitor", "base-url")
self.status_history_folder = os.path.join(self.public_folder, 'history')
......@@ -106,6 +107,7 @@ class Monitoring(object):
self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
self.config_folder = os.path.join(self.private_folder, 'config')
self.report_folder = os.path.join(self.private_folder, 'report')
self.promise_dict = {}
for promise_folder in self.promise_folder_list:
......@@ -267,9 +269,11 @@ class Monitoring(object):
raise
self.data_folder = os.path.join(self.private_folder, 'data', '.jio_documents')
self.report_folder = os.path.join(self.report_folder, '.jio_documents')
config_folder = os.path.join(self.config_folder, '.jio_documents')
mkdirAll(self.data_folder)
mkdirAll(config_folder)
mkdirAll(self.report_folder)
try:
os.symlink(os.path.join(self.private_folder, 'data'),
os.path.join(jio_private, 'data'))
......@@ -281,6 +285,11 @@ class Monitoring(object):
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
try:
os.symlink(self.report_folder, os.path.join(jio_private, 'report'))
except OSError, e:
if e.errno != os.errno.EEXIST:
raise
def makeConfigurationFiles(self):
config_folder = os.path.join(self.config_folder, '.jio_documents')
......@@ -385,6 +394,53 @@ class Monitoring(object):
with open(self.monitor_hal_json, "w") as fp:
json.dump(self.monitor_dict, fp)
def generateReportCronEntries(self):
cron_line_list = []
# We should add the possibility to modify this parameter later from monitor interface
report_frequency = "*/30 * * * *"
report_name_list = [name.replace('.report.json', '')
for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
for filename in os.listdir(self.report_script_folder):
report_script = os.path.join(self.report_script_folder, filename)
if os.path.isfile(report_script) and os.access(report_script, os.X_OK):
report_name = os.path.splitext(filename)[0]
report_json_path = "%s.report.json" % report_name
report_cmd_line = [
report_frequency,
self.promise_runner,
'--pid_path "%s"' % os.path.join(self.service_pid_folder,
"%s.pid" % filename),
'--output "%s"' % os.path.join(self.report_folder,report_json_path),
'--promise_script "%s"' % report_script,
'--promise_name "%s"' % report_name,
'--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder "%s"' % self.report_folder,
'--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title,
'--promise_type "report"']
cron_line_list.append(' '.join(report_cmd_line))
if report_name in report_name_list:
report_name_list.pop(report_name_list.index(report_name))
# cleanup removed report json result
if report_name_list != []:
for report_name in report_name_list:
result_path = os.path.join(self.public_folder, '%s.report.json' % report_name)
if os.path.exists(result_path):
try:
os.unlink(result_path)
except OSError, e:
print "Error: Failed to delete %s" % result_path, str(e)
pass
with open(self.crond_folder + "/monitor-reports", "w") as freport:
freport.write("\n".join(cron_line_list))
def generateServiceCronEntries(self):
# XXX only if at least one configuration file is modified, then write in the cron
#cron_line_list = ['PATH=%s\n' % os.environ['PATH']]
......@@ -401,13 +457,13 @@ class Monitoring(object):
promise_cmd_line = [
softConfigGet(service_config, "service", "frequency") or "* * * * *",
self.promise_runner,
'--pid_path %s' % os.path.join(self.service_pid_folder,
'--pid_path "%s"' % os.path.join(self.service_pid_folder,
"%s.pid" % service_name),
'--output %s' % service_status_path,
'--promise_script %s' % promise["path"],
'--output "%s"' % service_status_path,
'--promise_script "%s"' % promise["path"],
'--promise_name "%s"' % service_name,
'--monitor_url "%s/jio_private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder %s' % self.status_history_folder,
'--history_folder "%s"' % self.status_history_folder,
'--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title]
......@@ -442,6 +498,13 @@ class Monitoring(object):
cronf.write(entry_line)
def bootstrapMonitor(self):
# create symlinks from monitor.conf
self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
self.createSymlinksFromConfig(self.private_folder, self.private_path_list)
self.configureFolders()
# create symlinks from service configurations
self.promise_items = self.promise_dict.items()
for service_name, promise in self.promise_items:
......@@ -457,12 +520,6 @@ class Monitoring(object):
private_path_list.split(),
service_name)
# create symlinks from monitor.conf
self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
self.createSymlinksFromConfig(self.private_folder, self.private_path_list)
self.configureFolders()
# generate monitor.json
self.monitor_dict = {}
self.generateMonitorHalJson()
......@@ -474,6 +531,9 @@ class Monitoring(object):
# put promises to a cron file
self.generateServiceCronEntries()
# put report script to cron
self.generateReportCronEntries()
# Generate parameters files and scripts
self.makeConfigurationFiles()
......
......@@ -24,15 +24,18 @@ def parseArguments():
help='Promise script to execute.')
parser.add_argument('--promise_name',
help='Title to give to this promise.')
parser.add_argument('--promise_type',
default='status',
help='Type of promise to execute. [status, report].')
parser.add_argument('--monitor_url',
help='Monitor Instance website URL.')
parser.add_argument('--history_folder',
help='Path where old result file will be placed before generate a new json result file.')
parser.add_argument('--instance_name',
default='UNKNOW Software Instance',
default='UNKNOWN Software Instance',
help='Software Instance name.')
parser.add_argument('--hosting_name',
default='UNKNOW Hosting Subscription',
default='UNKNOWN Hosting Subscription',
help='Hosting Subscription name.')
return parser.parse_args()
......@@ -74,14 +77,16 @@ def main():
updateStatusHistoryFolder(
parser.promise_name,
parser.output,
parser.history_folder
parser.history_folder,
parser.promise_type
)
with open(parser.output, "w") as outputfile:
json.dump(status_json, outputfile)
os.remove(parser.pid_path)
def updateStatusHistoryFolder(name, status_file, history_folder):
def updateStatusHistoryFolder(name, status_file, history_folder, promise_type):
old_history_list = []
keep_item_amount = 25
history_path = os.path.join(history_folder, name, '.jio_documents')
if not os.path.exists(status_file):
return
......@@ -96,18 +101,19 @@ def updateStatusHistoryFolder(name, status_file, history_folder):
else: raise
with open(status_file, 'r') as sf:
status_dict = json.loads(sf.read())
filename = '%s.status.json' % (
status_dict['start-date'].replace(' ', '_').replace(':', ''))
filename = '%s.%s.json' % (
status_dict['start-date'].replace(' ', '_').replace(':', ''),
promise_type)
copyfile(status_file, os.path.join(history_path, filename))
# Don't let history foler grow too much, keep 40 files
# Don't let history foler grow too much, keep xx files
file_list = filter(os.path.isfile,
glob.glob("%s/*.status.json" % history_path)
glob.glob("%s/*.%s.json" % (history_path, promise_type))
)
file_count = len(file_list)
if file_count > 40:
if file_count > keep_item_amount:
file_list.sort(key=lambda x: os.path.getmtime(x))
while file_count > 40:
while file_count > keep_item_amount:
to_delete = file_list.pop(0)
try:
os.unlink(to_delete)
......
......@@ -63,7 +63,7 @@ mode = 0644
recipe = slapos.recipe.template
url = ${:_profile_base_location_}/instance-pull-backup.cfg.in
output = ${buildout:directory}/instance-pull-backup.cfg
md5sum = 028dfc01dfb0d738e1b4793f67b24e8c
md5sum = cb7acac7ab41bf44c20d6d03bfad8217
mode = 0644
[template-replicated]
......
......@@ -11,6 +11,7 @@ parts =
## Monitor for pbs
monitor-base
monitor-check-resilient-feed-file
extends = ${monitor2-template:rendered}
......@@ -266,7 +267,7 @@ private-path-list +=
[monitor-check-resilient-feed-file]
recipe = slapos.recipe.template:jinja2
template = ${template-monitor-check-resilient-feed:location}/${template-monitor-check-resilient-feed:filename}
rendered = $${monitor-directory:promises}/check-create-resilient-feed-files
rendered = $${monitor-directory:reports}/check-create-resilient-feed-files
mode = 700
context =
key input_feed_directory directory:notifier-feeds
......
......@@ -3,5 +3,5 @@
networkcache-section = networkcache
[networkcache]
download-cache-url = http://download.shacache.org/
download-dir-url = http://dir.shacache.org/
download-cache-url = http://www.shacache.org/shacache
download-dir-url = http://www.shacache.org/shadir
......@@ -194,8 +194,8 @@ uritemplate = 0.6
zope.interface = 4.1.3
[networkcache]
download-cache-url = http://download.shacache.org/
download-dir-url = http://dir.shacache.org/
download-cache-url = http://www.shacache.org/shacache
download-dir-url = http://www.shacache.org/shadir
# signature certificates of the following uploaders.
# Romain Courteaud
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment