Skip to content

S
slapos.toolbox
Commit ab39414c authored by Alain Takoudjou's avatar Alain Takoudjou
Browse files
Options

monitor: reimplement monitor following the new promise design in slapgrid 

promise are run by slapgrid now, monitor will collect result and generate final files
monitor runpromise script can be used to test a promise (which is implemented on the new promise design)
remove all generated script and logrotate code from monitor.py, they will be added in buildout
parent 53bfe1fe
Show whitespace changes
Inline Side-by-side
Showing with 731 additions and 1019 deletions
setup.py
View file @ ab39414c
......@@ -52,6 +52,7 @@ setup(name=name,
'PyRSS2Gen',
'dnspython',
'requests',
'jsonschema',
] + additional_install_requires,
extras_require = {
'lampconfigure': ["mysqlclient"], #needed for MySQL Database access
......
slapos/monitor/collect.py
View file @ ab39414c
......@@ -69,7 +69,7 @@ class ResourceCollect:
assert os.path.exists(db_path)
if db_path.endswith("collector.db"):
db_path = db_path[:-len("collector.db")]
self.db = Database(db_path)
self.db = Database(db_path, create=False)
self.consumption_utils = ConsumptionReportBase(self.db)
def has_table(self, name):
......@@ -255,7 +255,18 @@ def main():
initMemoryDataFile(mem_file)
initIODataFile(io_file)
with open(status_file, "w") as status_file:
status_file.write('{"cpu_time": 0, "cpu_percent": 0, "memory_rss": 0, "memory_percent": 0, "io_rw_counter": 0, "date": "", "total_process": 0, "disk_used": 0, "io_cycles_counter": 0, "cpu_num_threads": 0}')
status_file.write(json.dumps({
"cpu_time": 0,
"cpu_percent": 0,
"memory_rss": 0,
"memory_percent": 0,
"io_rw_counter": 0,
"date": "",
"total_process": 0,
"disk_used": 0,
"io_cycles_counter": 0,
"cpu_num_threads": 0
}))
with open(resource_file, "w") as resource_file:
resource_file.write('[]')
exit(1)
......
slapos/monitor/globalstate.py
View file @ ab39414c
......@@ -14,7 +14,7 @@ import PyRSS2Gen
def getKey(item):
return item.source.name
class monitorFeed(object):
class MonitorFeed(object):
def __init__(self, instance_name, hosting_name,
public_url, private_url, feed_url):
......@@ -26,23 +26,24 @@ class monitorFeed(object):
self.private_url = private_url
self.feed_url = feed_url
def appendItem(self, item_dict):
event_time = datetime.utcfromtimestamp(item_dict['change-time'])
description = item_dict.get('message', '')
def appendItem(self, item_dict, has_string=""):
event_date = item_dict['result']['change-date']
report_date = item_dict['result']['date']
description = item_dict['result'].get('message', '')
rss_item = PyRSS2Gen.RSSItem(
categories = [item_dict['status']],
source = PyRSS2Gen.Source(item_dict['title'], self.public_url),
title = '[%s] %s' % (item_dict['status'], item_dict['title']),
description = "%s: %s\n\n%s" % (event_time, item_dict['status'], description),
description = "\n%s" % (description,),
link = self.private_url,
pubDate = event_time,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s, %s" % (self.hosting_name,
item_dict['title'], event_time)),
pubDate = event_date,
guid = PyRSS2Gen.Guid(base64.b64encode("%s, %s, %s, %s" % (self.hosting_name,
item_dict['title'], has_string, event_date)),
isPermaLink=False)
)
self.rss_item_list.append(rss_item)
def genrss(self, output_file):
def generateRSS(self, output_file):
### Build the rss feed
# try to keep the list in the same order
sorted(self.rss_item_list, key=getKey)
......@@ -57,12 +58,6 @@ class monitorFeed(object):
with open(output_file, 'w') as frss:
frss.write(rss_feed.to_xml())
def softConfigGet(config, *args, **kwargs):
try:
return config.get(*args, **kwargs)
except (ConfigParser.NoOptionError, ConfigParser.NoSectionError):
return ""
def generateStatisticsData(stat_file_path, content):
# csv document for success/error statictics
if not os.path.exists(stat_file_path) or os.stat(stat_file_path).st_size == 0:
......@@ -89,63 +84,148 @@ def generateStatisticsData(stat_file_path, content):
fstat.seek(position)
fstat.write('%s}' % ',"{}"]'.format(current_state))
def run(args_list):
monitor_file, instance_file = args_list
monitor_config = ConfigParser.ConfigParser()
monitor_config.read(monitor_file)
base_folder = monitor_config.get('monitor', 'private-folder')
status_folder = monitor_config.get('monitor', 'public-folder')
base_url = monitor_config.get('monitor', 'base-url')
related_monitor_list = monitor_config.get("monitor", "monitor-url-list").split()
statistic_folder = os.path.join(base_folder, 'documents')
# need webdav to update parameters
parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
feed_output = os.path.join(status_folder, 'feed')
public_url = "%s/share/public/" % base_url
private_url = "%s/share/private/" % base_url
feed_url = "%s/public/feed" % base_url
def writeDocumentList(folder_path):
# Save document list in a file called _document_list
public_document_list = [os.path.splitext(file)[0]
for file in os.listdir(folder_path) if file.endswith('.json')]
error = success = 0
status = 'OK'
global_state_file = os.path.join(base_folder, 'monitor.global.json')
public_state_file = os.path.join(status_folder, 'monitor.global.json')
with open(os.path.join(folder_path, '_document_list'), 'w') as lfile:
lfile.write('\n'.join(public_document_list))
def generateMonitoringData(config, public_folder, private_folder, public_url,
private_url, feed_url):
feed_output = os.path.join(public_folder, 'feed')
# search for all status files
file_list = filter(os.path.isfile,
glob.glob("%s/*.status.json" % status_folder)
file_list = filter(
os.path.isfile,
glob.glob("%s/promise/*.status.json" % public_folder)
)
if os.path.exists(instance_file):
config = ConfigParser.ConfigParser()
config.read(instance_file)
else:
raise Exception("Cannot read instance configuration at %s" % instance_file)
report_date = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
monitor_feed = monitorFeed(
config.get('instance', 'name'),
config.get('instance', 'root-name'),
promises_status_file = os.path.join(private_folder, '_promise_status')
previous_state_dict = {}
new_state_dict = {}
error = success = 0
monitor_feed = MonitorFeed(
config.get('monitor', 'title'),
config.get('monitor', 'root-title'),
public_url,
private_url,
feed_url)
if os.path.exists(promises_status_file):
with open(promises_status_file) as f:
try:
previous_state_dict = json.loads(f.read())
except ValueError:
pass
for file in file_list:
try:
with open(file, 'r') as temp_file:
tmp_json = json.loads(temp_file.read())
except ValueError:
# bad json file ?
continue
if tmp_json['status'] == 'ERROR':
if tmp_json['result']['failed']:
promise_status = "ERROR"
error += 1
elif tmp_json['status'] == 'OK':
else:
promise_status = "OK"
success += 1
monitor_feed.appendItem(tmp_json)
tmp_json['result']['change-date'] = tmp_json['result']['date']
if previous_state_dict.has_key(tmp_json['name']):
status, change_date, _ = previous_state_dict[tmp_json['name']]
if promise_status == status:
tmp_json['result']['change-date'] = change_date
tmp_json['status'] = promise_status
message_hash = hashlib.md5(tmp_json['result'].get('message', '')).hexdigest()
new_state_dict[tmp_json['name']] = [
promise_status,
tmp_json['result']['change-date'],
message_hash
]
monitor_feed.appendItem(tmp_json, message_hash)
savePromiseHistory(
tmp_json['title'],
tmp_json,
previous_state_dict.get(tmp_json['name']),
public_folder
)
except ValueError, e:
# bad json file
print "ERROR: Bad json file at: %s\n%s" % (file, str(e))
continue
with open(promises_status_file, "w") as f:
json.dump(new_state_dict, f)
monitor_feed.generateRSS(feed_output)
return error, success
def savePromiseHistory(promise_name, state_dict, previous_state_list,
history_folder):
if not os.path.exists(history_folder) and os.path.isdir(history_folder):
self.logger.warning('Bad promise history folder, history is not saved...')
return
history_file = os.path.join(
history_folder,
'%s.history.json' % promise_name
)
# Remove useless informations
result = state_dict.pop('result')
state_dict.update(result)
state_dict.pop('path', '')
state_dict.pop('type', '')
if not os.path.exists(history_file) or not os.stat(history_file).st_size:
with open(history_file, 'w') as f:
data_dict = {
"date": time.time(),
"data": [state_dict]
}
json.dump(data_dict, f)
else:
if previous_state_list is not None:
_, change_date, checksum = previous_state_list
current_sum = hashlib.md5(state_dict.get('message', '')).hexdigest()
if state_dict['change-date'] == change_date and \
current_sum == checksum:
# Only save the changes and not the same info
return
state_dict.pop('title', '')
state_dict.pop('name', '')
with open (history_file, mode="r+") as f:
f.seek(0,2)
f.seek(f.tell() -2)
f.write('%s}' % ',{}]'.format(json.dumps(state_dict)))
def run(monitor_conf_file):
config = ConfigParser.ConfigParser()
config.read(monitor_conf_file)
base_folder = config.get('monitor', 'private-folder')
status_folder = config.get('monitor', 'public-folder')
base_url = config.get('monitor', 'base-url')
related_monitor_list = config.get("monitor", "monitor-url-list").split()
statistic_folder = os.path.join(base_folder, 'documents')
# need webdav to update parameters
parameter_file = os.path.join(base_folder, 'config', '.jio_documents', 'config.json')
public_url = "%s/share/public/" % base_url
private_url = "%s/share/private/" % base_url
feed_url = "%s/public/feed" % base_url
status = 'OK'
global_state_file = os.path.join(base_folder, 'monitor.global.json')
public_state_file = os.path.join(status_folder, 'monitor.global.json')
report_date = datetime.utcnow().strftime('%Y-%m-%dT%H:%M:%S')
error, success = generateMonitoringData(config, status_folder, base_folder,
public_url, private_url, feed_url)
if error:
status = 'ERROR'
monitor_feed.genrss(feed_output)
global_state_dict = dict(
status=status,
state={
......@@ -157,7 +237,8 @@ def run(args_list):
date=report_date,
_links={"rss_url": {"href": feed_url},
"public_url": {"href": public_url},
"private_url": {"href": private_url}
"private_url": {"href": private_url},
"related_monitor": []
},
data={'state': 'monitor_state.data',
'process_state': 'monitor_process_resource.status',
......@@ -165,25 +246,19 @@ def run(args_list):
'memory_resource': 'monitor_resource_memory.data',
'io_resource': 'monitor_resource_io.data',
'monitor_process_state': 'monitor_resource.status'},
_embedded={},
title=config.get('monitor', 'title'),
specialise_title=config.get('monitor', 'root-title'),
aggregate_reference=config.get('promises', 'computer-id'),
ipv4=config.get('promises', 'ipv4'),
ipv6=config.get('promises', 'ipv6'),
software_release=config.get('promises', 'software-release'),
software_type=config.get('promises', 'software-type'),
partition_id=config.get('promises', 'partition-id'),
)
instance_dict = {}
global_state_dict['title'] = config.get('instance', 'name')
global_state_dict['specialise_title'] = config.get('instance', 'root-name')
global_state_dict['aggregate_reference'] = config.get('instance', 'computer')
if not global_state_dict['title']:
global_state_dict['title'] = 'Instance Monitoring'
instance_dict['computer'] = config.get('instance', 'computer')
instance_dict['ipv4'] = config.get('instance', 'ipv4')
instance_dict['ipv6'] = config.get('instance', 'ipv6')
instance_dict['software-release'] = config.get('instance', 'software-release')
instance_dict['software-type'] = config.get('instance', 'software-type')
instance_dict['partition'] = config.get('instance', 'partition')
global_state_dict['_embedded'].update({'instance' : instance_dict})
if related_monitor_list:
global_state_dict['_links']['related_monitor'] = [{'href': "%s/share/public" % url}
for url in related_monitor_list]
......@@ -197,9 +272,9 @@ def run(args_list):
status=status,
date=report_date,
_links={'monitor': {'href': '%s/share/private/' % base_url}},
title=global_state_dict.get('title', '')
title=global_state_dict.get('title', ''),
specialise_title=global_state_dict.get('specialise_title', ''),
)
public_state_dict['specialise_title'] = global_state_dict.get('specialise_title', '')
public_state_dict['_links']['related_monitor'] = global_state_dict['_links'].get('related_monitor', [])
with open(global_state_file, 'w') as fglobal:
......@@ -208,31 +283,20 @@ def run(args_list):
with open(public_state_file, 'w') as fpglobal:
fpglobal.write(json.dumps(public_state_dict))
# Save document list in a file called _document_list
public_document_list = [os.path.splitext(file)[0]
for file in os.listdir(status_folder) if file.endswith('.json')]
private_document_list = [os.path.splitext(file)[0]
for file in os.listdir(base_folder) if file.endswith('.json')]
data_document_list = [os.path.splitext(file)[0]
for file in os.listdir(statistic_folder) if file.endswith('.json')]
with open(os.path.join(status_folder, '_document_list'), 'w') as lfile:
lfile.write('\n'.join(public_document_list))
with open(os.path.join(base_folder, '_document_list'), 'w') as lfile:
lfile.write('\n'.join(private_document_list))
with open(os.path.join(statistic_folder, '_document_list'), 'w') as lfile:
lfile.write('\n'.join(data_document_list))
# write list of files
writeDocumentList(status_folder)
writeDocumentList(base_folder)
writeDocumentList(statistic_folder)
generateStatisticsData(
os.path.join(statistic_folder, 'monitor_state.data.json'),
global_state_dict)
global_state_dict
)
return 0
def main():
if len(sys.argv) < 3:
print("Usage: %s <monitor_conf_path> <instance_conf_path>" % sys.argv[0])
if len(sys.argv) < 2:
print("Usage: %s <monitor_conf_path>" % sys.argv[0])
sys.exit(2)
sys.exit(run(sys.argv[1:]))
sys.exit(run(sys.argv[1]))
slapos/monitor/monitor.py
View file @ ab39414c
......@@ -36,7 +36,7 @@ def parseArguments():
Parse arguments for monitor instance.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--config_file',
parser.add_argument('-c', '--config-file', required=True,
default='monitor.cfg',
help='Monitor Configuration file')
......@@ -67,6 +67,7 @@ def createSymlink(source, destination):
class Monitoring(object):
def __init__(self, configuration_file):
self._config_file = configuration_file
config = self.loadConfig([configuration_file])
# Set Monitor variables
......@@ -74,19 +75,11 @@ class Monitoring(object):
self.root_title = config.get("monitor", "root-title")
self.service_pid_folder = config.get("monitor", "service-pid-folder")
self.crond_folder = config.get("monitor", "crond-folder")
self.logrotate_d = config.get("monitor", "logrotate-folder")
self.promise_runner = config.get("monitor", "promise-runner")
self.promise_folder = config.get("monitor", "promise-folder")
self.public_folder = config.get("monitor", "public-folder")
self.private_folder = config.get("monitor", "private-folder")
self.collector_db = config.get("monitor", "collector-db")
self.collect_script = config.get("monitor", "collect-script")
self.statistic_script = config.get("monitor", "statistic-script")
self.webdav_folder = config.get("monitor", "webdav-folder")
self.report_script_folder = config.get("monitor", "report-folder")
self.webdav_url = '%s/share' % config.get("monitor", "base-url")
self.public_url = '%s/public' % config.get("monitor", "base-url")
self.python = config.get("monitor", "python") or "python"
self.public_path_list = config.get("monitor", "public-path-list").split()
self.private_path_list = config.get("monitor", "private-path-list").split()
self.monitor_url_list = config.get("monitor", "monitor-url-list").split()
......@@ -94,25 +87,14 @@ class Monitoring(object):
# Use this file to write knowledge0_cfg required by webrunner
self.parameter_cfg_file = config.get("monitor", "parameter-file-path").strip()
self.pid_file = config.get("monitor", "pid-file")
self.monitor_promise_folder = softConfigGet(config, "monitor",
"monitor-promise-folder")
self.promise_timeout_file = softConfigGet(config, "monitor",
"promises-timeout-file")
self.nice_command = softConfigGet(config, "monitor",
"nice-cmd")
self.promise_output_file = config.get("monitor", "promise-output-file")
self.promise_folder = config.get("promises", 'promise-folder')
self.legacy_promise_folder = config.get("promises", 'legacy-promise-folder')
self.promise_output = config.get("promises", 'output-folder')
self.config_folder = os.path.join(self.private_folder, 'config')
self.report_folder = self.private_folder
self.data_folder = os.path.join(self.private_folder, 'documents')
self.log_folder = os.path.join(self.private_folder, 'monitor-log')
self.promise_output_file = config.get("monitor", "promise-output-file")
self.data_folder = config.get("monitor", "document-folder")
self.bootstrap_is_ok = True
self.randomsleep = config.get("monitor", "randomsleep")
if self.nice_command:
# run monitor promises script with low priority
self.promise_runner = '%s %s' % (self.nice_command, self.promise_runner)
def loadConfig(self, pathes, config=None):
if config is None:
......@@ -224,7 +206,7 @@ class Monitoring(object):
success = False
monitor_title = 'Unknown Instance'
try:
# Timeout after 20 seconds to not stall on download
# Timeout after 20 seconds
timeout = 20
# XXX - working here with public url
if hasattr(ssl, '_create_unverified_context'):
......@@ -247,28 +229,6 @@ class Monitoring(object):
self.bootstrap_is_ok = success
return monitor_title
def getReportInfoFromFilename(self, filename):
splited_filename = filename.split('_every_')
possible_time_list = ['hour', 'minute']
if len(splited_filename) == 1:
return (filename, "* * * * *")
run_time = splited_filename[1].split('_')
report_name = splited_filename[0]
if len(run_time) != 2 or not run_time[1] in possible_time_list:
return (report_name, "* * * * *")
try:
value = int(run_time[0])
except ValueError:
print "Warning: Bad report filename: %s" % filename
return (report_name, "* * * * *")
if run_time[1] == 'hour':
return (report_name, "11 */%s * * *" % value)
if run_time[1] == 'minute':
return (report_name, "*/%s * * * *" % value)
def configureFolders(self):
# create symlinks from monitor.conf
self.createSymlinksFromConfig(self.public_folder, self.public_path_list)
......@@ -278,19 +238,8 @@ class Monitoring(object):
self.createSymlinksFromConfig(self.webdav_folder, [self.private_folder])
config_jio_folder = os.path.join(self.config_folder, '.jio_documents')
mkdirAll(self.data_folder)
mkdirAll(config_jio_folder)
# Cleanup private folder, remove files that should not be synced from private root dir
cleanup_file_list = glob.glob("%s/*.history.json" % self.private_folder)
cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.private_folder))
for file in cleanup_file_list:
try:
os.unlink(file)
except OSError:
print "failed to remove file %s. Ignoring..." % file
def makeConfigurationFiles(self):
config_folder = os.path.join(self.config_folder, '.jio_documents')
parameter_config_file = os.path.join(config_folder, 'config.parameters.json')
......@@ -353,113 +302,43 @@ class Monitoring(object):
with open(output_file, 'w') as wfile:
wfile.write(opml_content)
def generateLogrotateEntry(self, name, file_list, option_list):
"""
Will add a new entry in logrotate.d folder. This can help to rotate data file daily
"""
content = "%(logfiles)s {\n%(options)s\n}\n" % {
'logfiles': ' '.join(file_list),
'options': '\n'.join(option_list)
}
file_path = os.path.join(self.logrotate_d, name)
with open(file_path, 'w') as flog:
flog.write(content)
def generateReportCronEntries(self):
cron_line_list = []
report_name_list = [name.replace('.report.json', '')
for name in os.listdir(self.report_folder) if name.endswith('.report.json')]
for filename in os.listdir(self.report_script_folder):
report_script = os.path.join(self.report_script_folder, filename)
if os.path.isfile(report_script) and os.access(report_script, os.X_OK):
report_name, frequency = self.getReportInfoFromFilename(filename)
report_json_name = "%s.report.json" % report_name
report_cmd_line = [
frequency,
self.promise_runner,
'--pid_path "%s"' % os.path.join(self.service_pid_folder,
"%s.pid" % filename),
'--output "%s"' % os.path.join(self.data_folder, report_json_name),
'--promise_script "%s"' % report_script,
'--promise_name "%s"' % report_name,
'--monitor_url "%s/private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder "%s"' % self.data_folder,
'--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title,
'--promise_type "report"',
'--log_file "%s.report.log"' % os.path.join(self.log_folder, report_name)]
cron_line_list.append(' '.join(report_cmd_line))
if report_name in report_name_list:
report_name_list.pop(report_name_list.index(report_name))
# cleanup removed report json result
if report_name_list != []:
for report_name in report_name_list:
result_path = os.path.join(self.report_folder, '%s.report.json' % report_name)
if os.path.exists(result_path):
def cleanupMonitorDeprecated(self):
# Monitor report feature is removed
cleanup_file_list = glob.glob("%s/*.history.json" % self.private_folder)
cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.private_folder))
cleanup_file_list.extend(glob.glob("%s/*.report.json" % self.data_folder))
cleanup_file_list.extend(glob.glob("%s/*.history.json" % self.data_folder))
cleanup_file_list.extend(glob.glob("%s/*.status.json" % self.public_folder))
cleanup_file_list.append(self.crond_folder + "/monitor-reports")
cleanup_file_list.append(self.crond_folder + "/monitor-promises")
for file in cleanup_file_list:
try:
os.unlink(result_path)
if os.path.exists(file):
os.unlink(file)
except OSError, e:
print "Error: Failed to delete %s" % result_path, str(e)
pass
print "failed to remove file %s." % file, str(e)
with open(self.crond_folder + "/monitor-reports", "w") as freport:
freport.write("\n".join(cron_line_list))
def generateServiceCronEntries(self):
# XXX only if at least one configuration file is modified, then write in the cron
service_name_list = [name.replace('.status.json', '')
for name in os.listdir(self.public_folder) if name.endswith('.status.json')]
promise_cmd_line = [
"* * * * *",
self.randomsleep + " 60 &&", # Sleep between 1 to 60 seconds
self.promise_runner,
'--pid_path "%s"' % os.path.join(self.service_pid_folder,
"monitor-promises.pid"),
'--output "%s"' % self.public_folder,
'--promise_folder "%s"' % self.promise_folder,
'--timeout_file "%s"' % self.promise_timeout_file,
'--monitor_promise_folder "%s"' % self.monitor_promise_folder,
'--monitor_url "%s/private/"' % self.webdav_url, # XXX hardcoded,
'--history_folder "%s"' % self.public_folder,
'--instance_name "%s"' % self.title,
'--hosting_name "%s"' % self.root_title,
'--log_file "%s.log"' % os.path.join(self.log_folder,
self.title.replace(' ', '_'))]
registered_promise_list = os.listdir(self.promise_folder)
registered_promise_list.extend(os.listdir(self.monitor_promise_folder))
delete_promise_list = []
for service_name in service_name_list:
if not service_name in registered_promise_list:
delete_promise_list.append(service_name)
if delete_promise_list != []:
# XXX Some service was removed, delete his status file so monitor will not consider the status anymore
for service_name in delete_promise_list:
status_path = os.path.join(self.public_folder, '%s.status.json' % service_name)
# cleanup result of promises that was removed
promise_list = os.listdir(self.legacy_promise_folder)
for name in os.listdir(self.promise_folder):
if name.startswith('__init__'):
continue
promise_list.append(os.path.splitext(name)[0])
for name in os.listdir(self.promise_output):
if not name.endswith('.status.json'):
continue
try:
position = promise_list.index(name.replace('.status.json', ''))
except ValueError:
status_path = os.path.join(self.promise_output, name)
if os.path.exists(status_path):
try:
os.unlink(status_path)
except OSError, e:
print "Error: Failed to delete %s" % status_path, str(e)
pass
with open(self.crond_folder + "/monitor-promises", "w") as fp:
fp.write(' '.join(promise_cmd_line))
def addCronEntry(self, name, frequency, command):
entry_line = '%s %s' % (frequency, command)
cron_entry_file = os.path.join(self.crond_folder, name)
with open(cron_entry_file, "w") as cronf:
cronf.write(entry_line)
else:
promise_list.pop(position)
def bootstrapMonitor(self):
......@@ -476,57 +355,12 @@ class Monitoring(object):
self.generateOpmlFile(self.monitor_url_list,
os.path.join(self.public_folder, 'feeds'))
# put promises to a cron file
self.generateServiceCronEntries()
# put report script to cron
self.generateReportCronEntries()
# cleanup deprecated entries
self.cleanupMonitorDeprecated()
# Generate parameters files and scripts
self.makeConfigurationFiles()
# Rotate monitor data files
option_list = [
'daily', 'nocreate', 'olddir %s' % self.data_folder, 'rotate 5',
'nocompress', 'missingok', 'extension .json', 'dateext',
'dateformat -%Y-%m-%d', 'notifempty'
]
file_list = [
"%s/*.data.json" % self.private_folder,
"%s/*.data.json" % self.data_folder]
self.generateLogrotateEntry('monitor.data', file_list, option_list)
# Rotate public history status file, delete data of previous days
option_list = [
'daily', 'nocreate', 'rotate 0',
'nocompress', 'notifempty', 'prerotate',
' %s --history_folder %s' % (self.statistic_script, self.public_folder),
'endscript'
]
file_list = ["%s/*.history.json" % self.public_folder]
self.generateLogrotateEntry('monitor.service.status', file_list, option_list)
# Rotate monitor log files
option_list = [
'daily', 'dateext', 'create', 'rotate 180',
'compress', 'delaycompress', 'notifempty',
'missingok'
]
file_list = ["%s/*.log" % self.log_folder]
self.generateLogrotateEntry('monitor.promise.log', file_list, option_list)
# Add cron entry for SlapOS Collect
command = self.randomsleep + " 60 && " # Random sleep between 1 to 60 seconds
if self.nice_command:
# run monitor collect with low priority
command += '%s ' % self.nice_command
command += "%s --output_folder %s --collector_db %s --pid_file %s" % (
self.collect_script,
self.data_folder,
self.collector_db,
os.path.join(self.service_pid_folder, "monitor-collect.pid"))
self.addCronEntry('monitor_collect', '* * * * *', command)
# Write an empty file when monitor bootstrap went until the end
if self.bootstrap_is_ok:
with open(self.promise_output_file, 'w') as promise_file:
......
slapos/monitor/runpromise.py
View file @ ab39414c
......@@ -13,147 +13,103 @@ import glob
import argparse
import traceback
import logging
from slapos.grid.utils import checkPromiseList as _checkPromiseList
import ConfigParser
from slapos.grid.promise import PromiseLauncher, PromiseQueueResult, PromiseError
from slapos.grid.promise.generic import PROMISE_LOG_FOLDER_NAME
from slapos.util import mkdir_p
# Promise timeout after 20 seconds by default
promise_timeout = 20
def parseArguments():
def getArgumentParser():
"""
Parse arguments for monitor collector instance.
"""
parser = argparse.ArgumentParser()
parser.add_argument('--pid_path',
parser.add_argument('-c', '--config', dest='config_file',
help='The Path of configuration file to load.')
parser.add_argument('-p', '--partition-folder',
help='Base path of the partition.')
parser.add_argument('-L', '--log-folder',
help='Folder where promises will write logs.')
parser.add_argument('--console',
help='Log to console too',
default=False, action='store_true')
parser.add_argument('-a', '--check-anomaly',
help='Enable check of anomaly on promises.',
default=False, action='store_true')
parser.add_argument('-D', '--debug',
help='Configure loggin in debug mode.',
default=False, action='store_true')
parser.add_argument('--master-url',
help='SlapOS Master Service URL.')
parser.add_argument('--partition-cert',
help='Computer partition certificate file path.')
parser.add_argument('--partition-key',
help='Computer partition key file path.')
parser.add_argument('--partition-id',
help='Computer partition ID.')
parser.add_argument('--computer-id',
help='Computer ID.')
parser.add_argument('--pid-path',
help='Path where the pid of this process will be writen.')
parser.add_argument('--output',
help='The Path of file where Json result of this promise will be saved.')
parser.add_argument('--promise_script',
help='Promise script to execute.')
parser.add_argument('--promise_folder',
help='Folder where to find promises to execute. Hide --promise_script and --promise_name')
parser.add_argument('--monitor_promise_folder',
help='Folder where to find Custom monitor promises to execute.')
parser.add_argument('--promise_name',
help='Title to give to this promise.')
parser.add_argument('--timeout_file',
default='',
help='File containing Max timeout for each promise run.')
parser.add_argument('--promise_type',
default='status',
help='Type of promise to execute. [status, report].')
parser.add_argument('--monitor_url',
help='Monitor Instance website URL.')
parser.add_argument('--history_folder',
help='Path where old result file will be placed before generate a new json result file.')
parser.add_argument('--instance_name',
default='UNKNOWN Software Instance',
help='Software Instance name.')
parser.add_argument('--hosting_name',
default='UNKNOWN Hosting Subscription',
help='Hosting Subscription name.')
parser.add_argument('--log_file',
help='Path of log file.')
parser.add_argument('--run-only',
help='List of python promises to run separates by space.',
default="")
parser.add_argument('-f', '--force',
help='Force to run promise without consider the periodicity.',
default=False, action='store_true')
parser.add_argument('--dry-run',
help='Only run promise without save result.',
default=False, action='store_true')
parser.add_argument('--promise-timeout', default=20, type=int,
help='Maximum promise execution time.')
return parser
class RunPromise(object):
class MonitorPromiseLauncher(object):
def __init__(self, config_parser):
self.config = config_parser
self.logger = logging.getLogger("RunPromise")
self.logger.setLevel(logging.DEBUG)
if self.config.config_file:
self._loadConfigFromFile(self.config.config_file)
self.logger = logging.getLogger("Monitor")
self.logger.setLevel(logging.DEBUG if self.config.debug else logging.INFO)
if not self.config.log_file:
if not self.config.log_folder or self.config.console:
if len(self.logger.handlers) == 0 or \
not isinstance(self.logger.handlers[0], logging.StreamHandler):
self.logger.addHandler(logging.StreamHandler())
else:
file_handler = logging.FileHandler(self.config.log_file)
handler = logging.StreamHandler()
handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
self.logger.addHandler(handler)
if self.config.log_folder:
log_file = os.path.join(self.config.log_folder,
'%s.log' % self.config.title.replace(' ', '_'))
file_handler = logging.FileHandler(log_file)
file_handler.setFormatter(logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s"))
self.logger.addHandler(file_handler)
self.promise_timeout = promise_timeout
if self.config.timeout_file and \
os.path.exists(self.config.timeout_file):
with open(self.config.timeout_file) as tf:
timeout = tf.read().strip()
if timeout.isdigit():
self.promise_timeout = int(timeout)
else:
self.logger.warning("%s it not a valid promise-timeout value" % timeout)
def runPromise(self):
if self.config.promise_folder:
# run all promises from the given folder in a synchronous way
return self.runPromiseSynchronous()
def _loadConfigFromFile(self, config_file):
config = ConfigParser.ConfigParser()
config.read([config_file])
known_key_list = ['partition-cert', 'partition-key', 'partition-id',
'pid-path', 'computer-id', 'check-anomaly',
'master-url', 'partition-folder', 'promise-timeout']
return self.runSinglePromise()
if config.has_section('promises'):
for key, value in config.items('promises'):
key_add = key.replace('-', '_')
if key in known_key_list and \
getattr(self.config, key_add, None) is None:
setattr(self.config, key_add, value)
def runSinglePromise(self):
"""
run a single promise in a new process
"""
if os.path.exists(self.config.pid_path):
with open(self.config.pid_path, "r") as pidfile:
try:
pid = int(pidfile.read(6))
except ValueError:
pid = None
if pid and os.path.exists("/proc/" + str(pid)):
self.logger.warning("A process is already running with pid " + str(pid))
return 1
start_date = datetime.utcnow()
with open(self.config.pid_path, "w") as pidfile:
self.logger.info("Running script %s..." % self.config.promise_script)
process = self.executeCommand(self.config.promise_script)
ps_process = psutil.Process(process.pid)
pidfile.write(str(process.pid))
status_json = self.generateStatusJsonFromProcess(
process,
start_date=start_date,
title=self.config.promise_name
)
status_json['_links'] = {"monitor": {"href": self.config.monitor_url}}
status_json['title'] = self.config.promise_name
status_json['instance'] = self.config.instance_name
status_json['hosting_subscription'] = self.config.hosting_name
status_json['type'] = self.config.promise_type
status_json['portal-type'] = "promise" if \
self.config.promise_type == "status" else self.config.promise_type
# Save the lastest status change date (needed for rss)
status_json['change-time'] = ps_process.create_time()
if os.path.exists(self.config.output) and os.stat(self.config.output).st_size:
with open(self.config.output) as f:
try:
last_result = json.loads(f.read())
if status_json['status'] == last_result['status'] and last_result.has_key('change-time'):
status_json['change-time'] = last_result['change-time']
except ValueError:
pass
self.updateStatusHistoryFolder(
self.config.promise_name,
self.config.output,
self.config.history_folder,
self.config.promise_type
)
# write the promise in a tmp file the move to promise output file
# this reduce conflict error on read/write at sametime
output_tmp = '%s.tmp' % self.config.output
with open(output_tmp, "w") as outputfile:
json.dump(status_json, outputfile)
os.rename(output_tmp, self.config.output)
os.remove(self.config.pid_path)
def runPromiseSynchronous(self):
def start(self):
"""
run all promises in sequential ways
"""
if self.config.pid_path:
if os.path.exists(self.config.pid_path):
# Check if another run promise is running
with open(self.config.pid_path) as fpid:
......@@ -168,171 +124,49 @@ class RunPromise(object):
with open(self.config.pid_path, 'w') as fpid:
fpid.write(str(os.getpid()))
promise_folder_list = [self.config.promise_folder]
if self.config.monitor_promise_folder:
promise_folder_list.append(self.config.monitor_promise_folder)
status_list = self.checkPromisesList(promise_folder_list)
promises_status_file = os.path.join(self.config.output, '_promise_status')
previous_state_dict = {}
new_state_dict = {}
base_dict = {
'_links': {"monitor": {"href": self.config.monitor_url}},
'instance': self.config.instance_name,
'hosting_subscription': self.config.hosting_name,
}
if os.path.exists(promises_status_file):
with open(promises_status_file) as f:
try:
previous_state_dict = json.loads(f.read())
except ValueError:
pass
for status_dict in status_list:
status_dict.update(base_dict)
if previous_state_dict.has_key(status_dict['title']):
status, change_time = previous_state_dict[status_dict['title']].split('#')
if status_dict['status'] == status:
status_dict['change-time'] = float(change_time)
promise_result_file = os.path.join(self.config.output,
"%s.status.json" % status_dict['title'])
# write the promise in a tmp file the move to promise output file
# this reduce conflict error on read/write at sametime
promise_tmp_file = '%s.tmp' % promise_result_file
with open(promise_tmp_file, "w") as outputfile:
json.dump(status_dict, outputfile)
os.rename(promise_tmp_file, promise_result_file)
new_state_dict[status_dict['title']] = '%s#%s' % (status_dict['status'],
status_dict['change-time'])
self.updateStatusHistoryFolder(
status_dict['title'],
promise_result_file,
self.config.history_folder,
'status'
)
with open(promises_status_file, "w") as outputfile:
json.dump(new_state_dict, outputfile)
os.remove(self.config.pid_path)
def updateStatusHistoryFolder(self, name, status_file, history_folder, promise_type):
history_path = os.path.join(history_folder)
if not os.path.exists(status_file):
return
if not os.path.exists(history_folder):
return
if not os.path.exists(history_path):
try:
os.makedirs(history_path)
except OSError, e:
if e.errno == os.errno.EEXIST and os.path.isdir(history_path):
pass
else: raise
with open(status_file, 'r') as sf:
try:
status_dict = json.loads(sf.read())
except ValueError:
traceback.print_exc()
return
if promise_type == 'status':
filename = '%s.history.json' % name
history_file = os.path.join(history_path, filename)
# Remove links from history (not needed)
status_dict.pop('_links', None)
if not os.path.exists(history_file) or \
(os.path.exists(history_file) and not os.stat(history_file).st_size):
with open(history_file, 'w') as f_history:
data_dict = {
"date": time.time(),
"data": [status_dict]
if not self.config.partition_folder:
raise ValueError("Partition folder is not specified")
parameter_dict = {
'promise-timeout': self.config.promise_timeout or promise_timeout,
'promise-folder': os.path.join(self.config.partition_folder, 'etc', 'plugin'),
'legacy-promise-folder': os.path.join(self.config.partition_folder, 'etc', 'promise'),
'partition-folder': self.config.partition_folder,
'master-url': self.config.master_url,
'partition-cert': self.config.partition_cert,
'partition-key': self.config.partition_key,
'partition-id': self.config.partition_id,
'computer-id': self.config.computer_id,
'debug': self.config.debug,
'check-anomaly': self.config.check_anomaly,
'force': self.config.force,
'run-only-promise-list': [x for x in self.config.run_only.split(' ') if x]
}
f_history.write(json.dumps(data_dict))
if self.config.log_folder:
parameter_dict['log-folder'] = self.config.log_folder
else:
# Remove useless informations
status_dict.pop('hosting_subscription', '')
status_dict.pop('title', '')
status_dict.pop('instance', '')
status_dict.pop('type', '')
status_dict.pop('portal-type', '')
with open (history_file, mode="r+") as f_history:
f_history.seek(0,2)
position = f_history.tell() -2
f_history.seek(position)
#f_history.write(',%s]}' % str(status_dict))
f_history.write('%s}' % ',{}]'.format(json.dumps(status_dict)))
elif promise_type == 'report':
# keep_item_amount = 3
filename = '%s.history.json' % (
name)
copyfile(status_file, os.path.join(history_path, filename))
def generateStatusJsonFromProcess(self, process, start_date=None, title=None):
stdout, stderr = process.communicate()
status_json = {}
if process.returncode != 0:
status_json["returncode"] = process.returncode
status_json["status"] = "ERROR"
if not stdout:
status_json["message"] = stderr
else:
status_json["message"] = stdout
else:
status_json["status"] = "OK"
status_json["message"] = stdout
status_json["returncode"] = 0
if start_date:
date_diff = (datetime.utcnow() - start_date)
status_json["start-date"] = start_date.strftime('%Y-%m-%dT%H:%M:%S')
status_json["execution-time"] = date_diff.total_seconds()
self.logger.info("Finished execution of %s in %s second(s)." % (
title, date_diff.total_seconds()))
if title:
status_json["title"] = title
return status_json
def executeCommand(self, args):
return subprocess.Popen(
args,
stdin=None,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE
parameter_dict['log-folder'] = os.path.join(self.config.partition_folder,
PROMISE_LOG_FOLDER_NAME)
mkdir_p(parameter_dict['log-folder'])
promise_launcher = PromiseLauncher(
config=parameter_dict,
logger=self.logger,
dry_run=self.config.dry_run
)
def checkPromisesList(self, promise_dir_list):
"""
Run all promises found into specified folder
"""
promise_list = []
self.logger.info("Checking promises...")
for promise_dir in promise_dir_list:
promise_list.extend(_checkPromiseList(
promise_dir,
self.promise_timeout,
profile=False,
raise_on_failure=False,
logger=self.logger))
# Check whether every promise is kept
for i in range(0, len(promise_list)):
promise_list[i]["status"] = "OK" if promise_list[i]["returncode"] == 0 else "ERROR"
promise_list[i]["type"] = "status"
promise_list[i]["portal-type"] = "promise"
promise_list[i]["change-time"] = time.mktime(promise_list[i]["start-date"].timetuple())
promise_list[i]["start-date"] = promise_list[i]["start-date"].strftime('%Y-%m-%dT%H:%M:%S')
try:
promise_launcher.run()
except PromiseError, e:
#self.logger.exception(e)
# error was already logged
pass
os.remove(self.config.pid_path)
self.logger.info("Finished promises.")
self.logger.info("---")
return promise_list
def main():
arg_parser = parseArguments()
promise_runner = RunPromise(arg_parser.parse_args())
sys.exit(promise_runner.runPromise())
arg_parser = getArgumentParser()
promise_runner = MonitorPromiseLauncher(arg_parser.parse_args())
sys.exit(promise_runner.start())
slapos/test/monitor/testbootstrap.py
View file @ ab39414c
......@@ -12,15 +12,18 @@ class MonitorBootstrapTest(unittest.TestCase):
def setUp(self):
self.base_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(self.base_dir, 'plugin'))
os.mkdir(os.path.join(self.base_dir, 'promise'))
os.mkdir(os.path.join(self.base_dir, 'monitor-promise'))
os.mkdir(os.path.join(self.base_dir, 'public'))
os.mkdir(os.path.join(self.base_dir, 'private'))
os.mkdir(os.path.join(self.base_dir, 'cron.d'))
os.mkdir(os.path.join(self.base_dir, 'log'))
os.mkdir(os.path.join(self.base_dir, 'log-backup'))
os.mkdir(os.path.join(self.base_dir, 'logrotate.d'))
os.mkdir(os.path.join(self.base_dir, 'monitor-report'))
os.mkdir(os.path.join(self.base_dir, 'webdav'))
os.mkdir(os.path.join(self.base_dir, 'run'))
os.mkdir(os.path.join(self.base_dir, 'private/documents'))
self.writeContent(os.path.join(self.base_dir, 'param'), '12345')
self.writeContent(os.path.join(self.base_dir, '.monitor_pwd'), 'bcuandjy')
self.writeContent(os.path.join(self.base_dir, 'test-httpd-cors.cfg'), '')
......@@ -28,13 +31,12 @@ class MonitorBootstrapTest(unittest.TestCase):
self.monitor_config_file = os.path.join(self.base_dir, 'monitor.conf')
self.monitor_config_dict = dict(
monitor_conf=self.monitor_config_file,
base_dir=self.base_dir,
root_title="Monitor ROOT",
title="Monitor",
url_list="",
base_url="https://monitor.test.com",
monitor_promise_folder=os.path.join(self.base_dir, 'monitor-promise'),
promise_folder=os.path.join(self.base_dir, 'promise'),
promise_runner_pid=os.path.join(self.base_dir, 'run', 'monitor-promises.pid'),
public_folder=os.path.join(self.base_dir, 'public'),
public_path_list="",
......@@ -45,18 +47,18 @@ class MonitorBootstrapTest(unittest.TestCase):
)
self.monitor_conf = """[monitor]
parameter-file-path = %(base_dir)s/knowledge0.cfg
promise-folder = %(base_dir)s/promise
service-pid-folder = %(base_dir)s/run
monitor-promise-folder = %(base_dir)s/monitor-promise
private-folder = %(base_dir)s/private
public-folder = %(base_dir)s/public
public-path-list = %(public_path_list)s
private-path-list = %(private_path_list)s
crond-folder = %(base_dir)s/cron.d
logrotate-folder = %(base_dir)s/logrotate.d
report-folder = %(base_dir)s/monitor-report
root-title = %(root_title)s
pid-file = %(base_dir)s/monitor.pid
log-folder = %(base_dir)s/log
log-backup-folder = %(base_dir)s/log-backup
document-folder = %(base_dir)s/private/documents
parameter-list =
raw monitor-user admin
file sample %(base_dir)s/param
......@@ -75,6 +77,12 @@ service-pid-folder = %(base_dir)s/run
promise-output-file = %(base_dir)s/monitor-bootstrap-status
promise-runner = %(promise_run_script)s
randomsleep = /bin/echo sleep
[promises]
output-folder = %(base_dir)s/public
legacy-promise-folder = %(base_dir)s/promise
promise-folder = %(base_dir)s/plugin
partition-folder = %(base_dir)s
"""
self.opml_outline = """<outline text="%(title)s" title="%(title)s" type="rss" version="RSS" htmlUrl="%(base_url)s/public/feed" xmlUrl="%(base_url)s/public/feed" url="%(base_url)s/share/private/" />"""
......@@ -87,21 +95,18 @@ randomsleep = /bin/echo sleep
with open(file_path, 'w') as cfg:
cfg.write(config)
def configPromises(self, amount):
def configPromiseList(self, amount):
promise_dir = os.path.join(self.base_dir, 'promise')
plugin_dir = os.path.join(self.base_dir, 'plugin')
promse_content = "/bin/bash echo something"
for index in range(1, amount+1):
promise_file = os.path.join(promise_dir, 'monitor_promise-%s' % index)
self.writeContent(promise_file, promse_content)
os.chmod(promise_file, 0755)
def configReports(self, amount):
promise_dir = os.path.join(self.base_dir, 'monitor-report')
promse_content = "/bin/bash echo something"
for index in range(1, amount+1):
promise_file = os.path.join(promise_dir, 'monitor_report-%s' % index)
promise_file = os.path.join(plugin_dir, 'monitor_promise-%s.py' % index)
self.writeContent(promise_file, promse_content)
os.chmod(promise_file, 0755)
os.chmod(promise_file, 0644)
def checkOPML(self, url_list):
opml_title = "<title>%(root_title)s</title>" % self.monitor_config_dict
......@@ -116,54 +121,6 @@ randomsleep = /bin/echo sleep
base_url=url)
self.assertTrue(opml_outline in opml_content)
def check_promises(self, sequential=False):
promise_cron = os.path.join(self.base_dir, 'cron.d', 'monitor-promises')
self.assertTrue(os.path.exists(promise_cron))
with open(promise_cron) as cronf:
promise_command_list = cronf.read()
if not sequential:
promise_entry = '* * * * * /bin/echo sleep 60 && %(promise_run_script)s --pid_path "%(promise_runner_pid)s" --output "%(public_folder)s" --promise_folder "%(promise_folder)s" --timeout_file "None" --monitor_promise_folder "%(monitor_promise_folder)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(base_dir)s/public" --instance_name "%(title)s" --hosting_name "%(root_title)s"'
entry_line = promise_entry % self.monitor_config_dict
self.assertTrue(entry_line in promise_command_list,
"%s not in %s" %(entry_line, promise_command_list))
else:
promise_entry = '* * * * * /bin/echo sleep 60 &&%(promise_run_script)s --pid_path "%(promise_pid)s" --output "%(promise_output)s" --promise_script "%(promise_executable)s" --promise_name "%(promise_name)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(base_dir)s/public" --instance_name "%(title)s" --hosting_name "%(root_title)s"'
promise_dir = os.path.join(self.base_dir, 'promise')
for filename in os.listdir(promise_dir):
promise_dict = dict(
promise_pid=os.path.join(self.base_dir, 'run', '%s.pid' % filename),
promise_output=os.path.join(self.base_dir, 'public', '%s.status.json' % filename),
promise_executable=os.path.join(promise_dir, filename),
promise_name=filename
)
promise_dict.update(self.monitor_config_dict)
entry_line = promise_entry % promise_dict
self.assertTrue(entry_line in promise_command_list)
def check_report(self):
promise_entry = '* * * * * %(promise_run_script)s --pid_path "%(promise_pid)s" --output "%(promise_output)s" --promise_script "%(promise_executable)s" --promise_name "%(promise_name)s" --monitor_url "%(base_url)s/share/private/" --history_folder "%(data_dir)s" --instance_name "%(title)s" --hosting_name "%(root_title)s" --promise_type "report"'
promise_dir = os.path.join(self.base_dir, 'monitor-report')
data_dir = os.path.join(self.base_dir, 'private', 'documents')
promise_cron = os.path.join(self.base_dir, 'cron.d', 'monitor-reports')
self.assertTrue(os.path.exists(promise_cron))
with open(promise_cron) as cronf:
promise_command_list = cronf.read()
for filename in os.listdir(promise_dir):
promise_dict = dict(
promise_pid=os.path.join(self.base_dir, 'run', '%s.pid' % filename),
promise_output=os.path.join(data_dir, '%s.report.json' % filename),
promise_executable=os.path.join(promise_dir, filename),
promise_name=filename,
data_dir=data_dir
)
promise_dict.update(self.monitor_config_dict)
entry_line = promise_entry % promise_dict
self.assertTrue(entry_line in promise_command_list)
def check_folder_equals(self, source, destination):
self.assertTrue(os.path.isdir(source))
......@@ -261,7 +218,7 @@ randomsleep = /bin/echo sleep
self.assertTrue(os.path.exists(os.path.join(self.base_dir, 'private/documents')))
def test_monitor_bootstrap_promises(self):
self.configPromises(3)
self.configPromiseList(3)
config_content = self.monitor_conf % self.monitor_config_dict
self.writeContent(self.monitor_config_file, config_content)
......@@ -273,38 +230,12 @@ randomsleep = /bin/echo sleep
self.checkOPML([self.monitor_config_dict['base_url']])
self.check_promises()
# Check update promises_list
self.configPromises(5) # Add two promises
self.configPromiseList(5) # Add two promises
os.unlink(os.path.join(self.base_dir, 'promise', 'monitor_promise-2')) # drop promise number 2
instance2 = Monitoring(self.monitor_config_file)
instance2.bootstrapMonitor()
self.assertTrue(os.path.exists(promise_file))
self.check_promises()
def test_monitor_bootstrap_report(self):
self.configReports(3)
config_content = self.monitor_conf % self.monitor_config_dict
self.writeContent(self.monitor_config_file, config_content)
instance = Monitoring(self.monitor_config_file)
instance.bootstrapMonitor()
promise_file = os.path.join(self.base_dir, 'monitor-bootstrap-status')
self.assertTrue(os.path.exists(promise_file))
self.checkOPML([self.monitor_config_dict['base_url']])
self.check_report()
# Check update promises_list
self.configReports(5) # Add two promises
os.unlink(os.path.join(self.base_dir, 'monitor-report', 'monitor_report-1')) # drop promise number 2
instance2 = Monitoring(self.monitor_config_file)
instance2.bootstrapMonitor()
self.assertTrue(os.path.exists(promise_file))
self.check_report()
def test_monitor_bootstrap_genconfig(self):
config_content = self.monitor_conf % self.monitor_config_dict
......
slapos/test/monitor/testglobalstate.py
View file @ ab39414c
......@@ -5,50 +5,56 @@ import shutil
import tempfile
import unittest
import json
import pkg_resources
from slapos.monitor import globalstate
from slapos.monitor.runpromise import RunPromise, parseArguments
from slapos.monitor.runpromise import MonitorPromiseLauncher, getArgumentParser
from slapos.monitor.monitor import Monitoring
from jsonschema import validate
class MonitorGlobalTest(unittest.TestCase):
def setUp(self):
self.base_dir = tempfile.mkdtemp()
os.mkdir(os.path.join(self.base_dir, 'promise'))
os.mkdir(os.path.join(self.base_dir, 'monitor-promise'))
self.etc_dir = os.path.join(self.base_dir, 'etc')
self.output_dir = os.path.join(self.base_dir, '.slapgrid/promise/result')
os.mkdir(self.etc_dir)
os.mkdir(os.path.join(self.etc_dir, 'plugin'))
os.mkdir(os.path.join(self.etc_dir, 'promise'))
os.mkdir(os.path.join(self.base_dir, 'public'))
os.mkdir(os.path.join(self.base_dir, 'private'))
os.mkdir(os.path.join(self.base_dir, 'cron.d'))
os.mkdir(os.path.join(self.base_dir, 'log'))
os.mkdir(os.path.join(self.base_dir, 'log-backup'))
os.mkdir(os.path.join(self.base_dir, 'logrotate.d'))
os.mkdir(os.path.join(self.base_dir, 'monitor-report'))
os.mkdir(os.path.join(self.base_dir, 'webdav'))
os.mkdir(os.path.join(self.base_dir, 'run'))
os.mkdir(os.path.join(self.base_dir, 'private/documents'))
self.writeContent(os.path.join(self.base_dir, 'param'), '12345')
self.writeContent(os.path.join(self.base_dir, '.monitor_pwd'), 'bcuandjy')
self.writeContent(os.path.join(self.base_dir, 'test-httpd-cors.cfg'), '')
self.writeContent(os.path.join(self.base_dir, 'monitor-htpasswd'), '12345')
self.writeContent(os.path.join(self.base_dir, 'instance.cfg'), """[instance]
name = Monitor
root-name = Monitor ROOT
computer = COMP-1234
ipv4 = 10.0.151.118
ipv6 = 2001:34c:1254:df3:89::5df3
software-release = http://some.url.com/software.cfg
software-type = default
partition = slappart10""")
self.monitor_config_file = os.path.join(self.base_dir, 'monitor.conf')
self.public_dir = os.path.join(self.base_dir, 'public')
self.private_dir = os.path.join(self.base_dir, 'private')
monitor_schema_string = \
pkg_resources.resource_string(
'slapos.monitor',
'doc/monitor_instance.schema.json')
self.monitor_instance_schema = json.loads(monitor_schema_string)
self.monitor_config_dict = dict(
monitor_conf=self.monitor_config_file,
base_dir=self.base_dir,
root_title="Monitor ROOT",
title="Monitor",
url_list="",
url_list="https://sub.monitor.test.com https://sub2.monitor.test.com",
base_url="https://monitor.test.com",
monitor_promise_folder=os.path.join(self.base_dir, 'monitor-promise'),
promise_folder=os.path.join(self.base_dir, 'promise'),
etc_dir=self.etc_dir,
promise_runner_pid=os.path.join(self.base_dir, 'run', 'monitor-promises.pid'),
public_folder=os.path.join(self.base_dir, 'public'),
public_path_list="",
......@@ -59,18 +65,18 @@ partition = slappart10""")
)
self.monitor_conf = """[monitor]
parameter-file-path = %(base_dir)s/knowledge0.cfg
promise-folder = %(base_dir)s/promise
service-pid-folder = %(base_dir)s/run
monitor-promise-folder = %(base_dir)s/monitor-promise
private-folder = %(base_dir)s/private
public-folder = %(base_dir)s/public
public-path-list = %(public_path_list)s
private-path-list = %(private_path_list)s
crond-folder = %(base_dir)s/cron.d
logrotate-folder = %(base_dir)s/logrotate.d
report-folder = %(base_dir)s/monitor-report
root-title = %(root_title)s
pid-file = %(base_dir)s/monitor.pid
log-folder = %(base_dir)s/log
log-backup-folder = %(base_dir)s/log-backup
document-folder = %(base_dir)s/private/documents
parameter-list =
raw monitor-user admin
file sample %(base_dir)s/param
......@@ -89,6 +95,21 @@ service-pid-folder = %(base_dir)s/run
promise-output-file = %(base_dir)s/monitor-bootstrap-status
promise-runner = %(promise_run_script)s
randomsleep = /bin/echo sleep
[promises]
output-folder = %(base_dir)s/public
legacy-promise-folder = %(etc_dir)s/promise
promise-folder = %(etc_dir)s/plugin
partition-folder = %(base_dir)s
computer-id = COMP-1234
partition-cert =
partition-key =
partition-id = slappart0
ipv6 = 2001:34c:1254:df3:89::5df3
software-release = http://some.url.com/software.cfg
master-url = http://10.0.151.118:50000
software-type = default
ipv4 = 10.0.151.118
"""
def tearDown(self):
......@@ -109,7 +130,7 @@ randomsleep = /bin/echo sleep
echo "%(output)s"
exit %(code)s
""" % result_dict
promise_path = os.path.join(self.base_dir, 'promise', name)
promise_path = os.path.join(self.etc_dir, 'promise', name)
self.writeContent(promise_path, content)
os.chmod(promise_path, 0755)
return promise_path
......@@ -118,18 +139,13 @@ exit %(code)s
pid_path = os.path.join(self.base_dir, 'run', 'monitor-promise.pid')
promise_cmd = [
'--pid_path',
'%s' % pid_path, '--output', os.path.join(self.base_dir, 'public'),
'--promise_folder', os.path.join(self.base_dir, 'promise'),
'--monitor_promise_folder', os.path.join(self.base_dir, 'monitor-promise'),
'--promise_type', 'status',
'--monitor_url', 'https://monitor.test.com/share/private/',
'--history_folder', os.path.join(self.base_dir, 'public'),
'--instance_name', 'Monitor', '--hosting_name', 'Monitor ROOT']
arg_parser = parseArguments()
'--pid-path',
'%s' % pid_path, '-c', self.monitor_config_file]
arg_parser = getArgumentParser()
return arg_parser.parse_args(promise_cmd)
def test_monitor_instance_state(self):
self.maxDiff = None
config_content = self.monitor_conf % self.monitor_config_dict
self.writeContent(self.monitor_config_file, config_content)
......@@ -141,31 +157,27 @@ exit %(code)s
self.writePromise('promise_3', success=False)
self.writePromise('promise_4')
parser = self.getPromiseParser()
promise_runner = RunPromise(parser)
promise_runner.runPromise()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_1.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_2.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_3.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'promise_4.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_1.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_2.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_3.status.json')))
self.assertTrue(os.path.exists(os.path.join(self.output_dir, 'promise_4.status.json')))
os.symlink(self.output_dir, '%s/public/promise' % self.base_dir)
# generate instance state files
globalstate.run([self.monitor_config_file, os.path.join(self.base_dir, 'instance.cfg')])
globalstate.run(self.monitor_config_file)
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'feed')))
self.assertTrue(os.path.exists(os.path.join(self.public_dir, 'monitor.global.json')))
self.assertTrue(os.path.exists(os.path.join(self.private_dir, 'monitor.global.json')))
expected_result = """{
"status": "ERROR",
"_embedded": {
"instance": {
"partition": "slappart10",
"partition_id": "slappart0",
"ipv6": "2001:34c:1254:df3:89::5df3",
"computer": "COMP-1234",
"ipv4": "10.0.151.118",
"software-release": "http://some.url.com/software.cfg",
"software-type": "default"
}
},
"software_release": "http://some.url.com/software.cfg",
"software_type": "default",
"parameters": [{
"key": "",
"value": "admin",
......@@ -206,7 +218,11 @@ exit %(code)s
},
"private_url": {
"href": "https://monitor.test.com/share/private/"
}
},
"related_monitor": [
{"href": "https://sub.monitor.test.com/share/public"},
{"href": "https://sub2.monitor.test.com/share/public"}
]
},
"aggregate_reference": "COMP-1234",
"type": "global",
......@@ -222,15 +238,22 @@ exit %(code)s
# all promises are OK now
self.writePromise('promise_2', success=True)
self.writePromise('promise_3', success=True)
promise_runner.runPromise()
globalstate.run([self.monitor_config_file, os.path.join(self.base_dir, 'instance.cfg')])
# rerun right now
promise_runner.config.force = True
promise_runner.start()
globalstate.run(self.monitor_config_file)
expected_result_dict = json.loads(expected_result)
expected_result_dict["status"] = "OK"
expected_result_dict["state"] = {'error': 0, 'success': 4}
instance_result_dict = None
with open(os.path.join(self.private_dir, 'monitor.global.json')) as r:
result = json.loads(r.read().decode("utf-8"))
instance_result_dict = json.loads(r.read().decode("utf-8"))
result = instance_result_dict.copy()
result.pop("date")
self.assertEquals(result,
expected_result_dict)
# validate returned json result
validate(instance_result_dict, self.monitor_instance_schema)
slapos/test/monitor/testrunpromise.py
View file @ ab39414c
......@@ -6,39 +6,68 @@ import tempfile
import unittest
import json
from datetime import datetime
from slapos.monitor.runpromise import RunPromise, parseArguments
from slapos.monitor.runpromise import MonitorPromiseLauncher, getArgumentParser
from slapos.grid.promise.generic import PROMISE_LOG_FOLDER_NAME, PROMISE_RESULT_FOLDER_NAME
class MonitorPromiseTest(unittest.TestCase):
def setUp(self):
self.base_dir = tempfile.mkdtemp()
self.promise_dir = os.path.join(self.base_dir, 'promise')
self.monitor_promise_dir = os.path.join(self.base_dir, 'monitor-promise')
self.report_dir = os.path.join(self.base_dir, 'report')
self.etc_dir = os.path.join(self.base_dir, 'etc')
self.output_dir = os.path.join(self.base_dir, PROMISE_RESULT_FOLDER_NAME)
self.log_output_dir = os.path.join(self.base_dir, PROMISE_LOG_FOLDER_NAME)
self.promise_dir = os.path.join(self.etc_dir, 'plugin')
self.old_promise_dir = os.path.join(self.etc_dir, 'promise')
self.public_dir = os.path.join(self.base_dir, 'public')
self.private_dir = os.path.join(self.base_dir, 'private')
self.run_dir = os.path.join(self.base_dir, 'run')
self.log_dir = os.path.join(self.base_dir, 'log')
os.mkdir(self.etc_dir)
os.mkdir(self.promise_dir)
os.mkdir(self.public_dir)
os.mkdir(self.private_dir)
os.mkdir(self.report_dir)
os.mkdir(self.run_dir)
os.mkdir(self.monitor_promise_dir)
os.mkdir(self.old_promise_dir)
os.mkdir(self.log_dir)
self.monitor_config = """[monitor]
private-folder = %(base_dir)s/private
public-folder = %(base_dir)s/public
root-title = %(root_title)s
log-folder = %(base_dir)s/log
base-url = %(base_url)s
title = %(title)s
[promises]
pid-path = %(base_dir)s/run/promises.pid
partition-folder = %(base_dir)s
computer-id = COMP-1234
partition-cert =
partition-key =
partition-id = slappart0
ipv6 = 2001:34c:1254:df3:89::5df3
software-release = http://some.url.com/software.cfg
master-url = http://10.0.151.118:50000
software-type = default
ipv4 = 10.0.151.118
""" % {'base_dir': self.base_dir, 'title': 'Monitor', 'root_title': 'Monitor ROOT',
'base_url': 'https://monitor.test.com', }
self.monitor_config_file = os.path.join(self.base_dir, 'monitor.conf')
with open(self.monitor_config_file, 'w') as f:
f.write(self.monitor_config)
def tearDown(self):
if os.path.exists(self.base_dir):
shutil.rmtree(self.base_dir)
def writePromiseOK(self, name, monitor_folder=False):
def writePromiseOK(self, name):
content = """#!/bin/sh
echo "success"
exit 0
"""
promise_path = os.path.join(self.promise_dir, name)
if monitor_folder:
promise_path = os.path.join(self.monitor_promise_dir, name)
promise_path = os.path.join(self.old_promise_dir, name)
self.writeContent(promise_path, content)
os.chmod(promise_path, 0755)
return promise_path
......@@ -49,348 +78,333 @@ exit 0
echo "failed"
exit 2
"""
promise_path = os.path.join(self.promise_dir, name)
promise_path = os.path.join(self.old_promise_dir, name)
self.writeContent(promise_path, content)
os.chmod(promise_path, 0755)
return promise_path
def generatePromiseScript(self, name, success=True, failure_count=1, content="",
periodicity=0.03):
promise_content = """from zope import interface as zope_interface
from slapos.grid.promise import interface
from slapos.grid.promise import GenericPromise
class RunPromise(GenericPromise):
zope_interface.implements(interface.IPromise)
def __init__(self, config):
GenericPromise.__init__(self, config)
self.setPeriodicity(minute=%(periodicity)s)
def sense(self):
%(content)s
if not %(success)s:
self.logger.error("failed")
else:
self.logger.info("success")
def anomaly(self):
return self._anomaly(result_count=2, failure_amount=%(failure_amount)s)
def test(self):
return self._test(result_count=1, failure_amount=%(failure_amount)s)
""" % {'success': success, 'content': content, 'failure_amount': failure_count,
'periodicity': periodicity}
with open(os.path.join(self.promise_dir, name), 'w') as f:
f.write(promise_content)
return os.path.join(self.promise_dir, name)
def writeContent(self, file_path, config):
with open(file_path, 'w') as cfg:
cfg.write(config)
def getUniquePromiseParser(self, name, promise_path, promise_type):
pid_path = os.path.join(self.run_dir, '%s.pid' % name)
if promise_type == "report":
output_path = os.path.join(self.private_dir, '%s.report.json' % name)
else:
output_path = os.path.join(self.public_dir, '%s.status.json' % name)
promise_cmd = [
'--pid_path',
'%s' % pid_path, '--output', output_path,
'--promise_script', promise_path,
'--promise_name', name, '--promise_type', promise_type,
'--monitor_url', 'https://monitor.test.com/share/private/',
'--history_folder', self.public_dir,
'--instance_name', 'Monitor', '--hosting_name', 'Monitor ROOT']
arg_parser = parseArguments()
return arg_parser.parse_args(promise_cmd)
def getPromiseParser(self, use_config=True, check_anomaly=False, force=False):
def getPromiseParser(self):
pid_path = os.path.join(self.run_dir, 'monitor-promise.pid')
arg_parser = getArgumentParser()
base_list = ['-c', self.monitor_config_file]
if use_config:
if check_anomaly:
base_list.append('-a')
if force:
base_list.append('--force')
return arg_parser.parse_args(base_list)
pid_path = os.path.join(self.run_dir, 'monitor-promise.pid')
promise_cmd = [
'--pid_path',
'%s' % pid_path, '--output', self.public_dir,
'--promise_folder', self.promise_dir,
'--monitor_promise_folder', self.monitor_promise_dir,
'--promise_type', 'status',
'--monitor_url', 'https://monitor.test.com/share/private/',
'--history_folder', self.public_dir,
'--instance_name', 'Monitor', '--hosting_name', 'Monitor ROOT']
arg_parser = parseArguments()
'--pid-path',
'%s' % pid_path, '-o', self.public_dir,
'--master-url', 'http://10.0.151.118:50000',
'--partition-id', 'slappart0',
'--computer-id', 'COMP-1234']
return arg_parser.parse_args(promise_cmd)
def test_promise_OK(self):
promise = self.writePromiseOK('promise_1')
def test_promise_generic(self):
promise_file = self.generatePromiseScript('my_promise.py', success=True)
promise_file2 = self.generatePromiseScript('my_second_promise.py', success=True)
parser = self.getPromiseParser()
promise_runner = RunPromise(parser)
promise_runner.runPromise()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result_file = os.path.join(self.public_dir, 'promise_1.status.json')
result_file = os.path.join(self.output_dir, 'my_promise.status.json')
os.system('cat %s' % result_file)
self.assertTrue(os.path.exists(result_file))
result1 = json.loads(open(result_file).read().decode("utf-8"))
change_time = result1.pop('change-time', 0)
change_date = datetime.fromtimestamp(change_time)
execution_time = result1.pop('execution-time')
start_date = result1.pop('start-date')
expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
u'title': u'promise_1', u'instance': u'Monitor',
u'_links':
{u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
u'message': u'success\n', u'type': u'status',
u'portal-type': u'promise', u'returncode': 0}
self.assertEquals(expected_result, result1)
# second run
time.sleep(1)
promise_runner.runPromise()
result2 = json.loads(open(result_file).read().decode("utf-8"))
change_time2 = result2.pop('change-time', 0)
start_date2 = result2.pop('start-date')
change_date2 = datetime.fromtimestamp(change_time2)
execution_time2 = result2.pop('execution-time')
self.assertEquals(expected_result, result2)
self.assertEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
change_date2.strftime('%Y-%m-%d %H:%M:%S'))
history_file = os.path.join(self.public_dir, 'promise_1.history.json')
self.assertTrue(os.path.exists(history_file))
history = json.load(open(history_file))
self.assertTrue(history['date'] > change_time)
self.assertTrue(len(history['data']) == 2)
result1['change-time'] = change_time
result1['start-date'] = start_date
result1.pop('_links')
result1['execution-time'] = execution_time
result2['start-date'] = start_date2
result2['change-time'] = change_time2
result2['execution-time'] = execution_time2
# not in history
result2.pop('_links')
result2.pop('hosting_subscription')
result2.pop('title')
result2.pop('instance')
result2.pop('type')
result2.pop('portal-type')
self.assertEquals(history['data'][0], result1)
self.assertEquals(history['data'][1], result2)
def test_promise_two_folder(self):
promise = self.writePromiseOK('promise_1')
promise2 = self.writePromiseOK('promise_2', monitor_folder=True)
my_result = json.loads(open(result_file).read().decode("utf-8"))
my_result['result'].pop('date')
expected_result = {
u'title': u'my_promise', u'name': u'my_promise.py',
u'result': {
u'failed': False, u'message': u'success', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s/my_promise.py' % self.promise_dir,
}
self.assertEquals(expected_result, my_result)
result_file = os.path.join(self.output_dir, 'my_second_promise.status.json')
self.assertTrue(os.path.exists(result_file))
second_result = json.loads(open(result_file).read().decode("utf-8"))
second_result['result'].pop('date')
expected_result = {
u'title': u'my_second_promise', u'name': u'my_second_promise.py',
u'result': {
u'failed': False, u'message': u'success', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s/my_second_promise.py' % self.promise_dir,
}
self.assertEquals(expected_result, second_result)
def test_promise_generic_failed(self):
promise_file = self.generatePromiseScript('my_promise.py', success=False)
parser = self.getPromiseParser()
promise_runner = RunPromise(parser)
promise_runner.runPromise()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result_file = os.path.join(self.public_dir, 'promise_1.status.json')
result2_file = os.path.join(self.public_dir, 'promise_2.status.json')
result_file = os.path.join(self.output_dir, 'my_promise.status.json')
self.assertTrue(os.path.exists(result_file))
self.assertTrue(os.path.exists(result2_file))
result1 = json.loads(open(result_file).read().decode("utf-8"))
result1.pop('change-time')
result1.pop('start-date')
self.assertTrue(result1.pop('execution-time', None) is not None)
expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
u'title': u'promise_1', u'instance': u'Monitor',
u'_links':
{u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
u'message': u'success\n', u'type': u'status',
u'portal-type': u'promise', u'returncode': 0}
self.assertEquals(expected_result, result1)
my_result = json.loads(open(result_file).read().decode("utf-8"))
my_result['result'].pop('date')
expected_result = {
u'title': u'my_promise', u'name': u'my_promise.py',
u'result': {
u'failed': True, u'message': u'failed', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s/my_promise.py' % self.promise_dir,
}
self.assertEquals(expected_result, my_result)
def test_promise_generic_status_change(self):
promise_file = self.generatePromiseScript('my_promise.py', success=False)
parser = self.getPromiseParser()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result2 = json.loads(open(result2_file).read())
result2.pop('change-time')
result2.pop('start-date')
self.assertTrue(result2.pop('execution-time', None) is not None)
expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
u'title': u'promise_2', u'instance': u'Monitor',
u'_links':
{u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
u'message': u'success\n', u'type': u'status',
u'portal-type': u'promise', u'returncode': 0}
self.assertEquals(expected_result, result2)
result_file = os.path.join(self.output_dir, 'my_promise.status.json')
self.assertTrue(os.path.exists(result_file))
my_result = json.loads(open(result_file).read().decode("utf-8"))
my_result['result'].pop('date')
expected_result = {
u'title': u'my_promise', u'name': u'my_promise.py',
u'result': {
u'failed': True, u'message': u'failed', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s/my_promise.py' % self.promise_dir,
}
self.assertEquals(expected_result, my_result)
if "my_promise" in sys.modules:
# force to reload the module without rerun python
os.system('rm %s/*.pyc' % self.promise_dir)
del sys.modules["my_promise"]
time.sleep(1.5)
self.generatePromiseScript('my_promise.py', success=True, periodicity=0.017)
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
my_result = json.loads(open(result_file).read().decode("utf-8"))
my_result['result'].pop('date')
expected_result = {
u'title': u'my_promise', u'name': u'my_promise.py',
u'result': {
u'failed': False, u'message': u'success', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s/my_promise.py' % self.promise_dir,
}
self.assertEquals(expected_result, my_result)
def test_promise_generic_periodicity(self):
promise_file = self.generatePromiseScript('my_promise.py', success=False, periodicity=0.03)
parser = self.getPromiseParser()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
def test_promise_One_By_One(self):
result_file = os.path.join(self.output_dir, 'my_promise.status.json')
self.assertTrue(os.path.exists(result_file))
promise = self.writePromiseOK('promise_1')
parser = self.getUniquePromiseParser('promise_1', promise, 'status')
promise_runner = RunPromise(parser)
promise_runner.runPromise()
# run again
os.unlink(result_file)
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
self.assertFalse(os.path.exists(result_file))
result_file = os.path.join(self.public_dir, 'promise_1.status.json')
time.sleep(2)
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
self.assertTrue(os.path.exists(result_file))
result1 = json.loads(open(result_file).read().decode("utf-8"))
change_time = result1.pop('change-time', 0)
change_date = datetime.fromtimestamp(change_time)
execution_time = result1.pop('execution-time', None)
start_date = result1.pop('start-date')
expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
u'title': u'promise_1', u'instance': u'Monitor',
u'_links':
{u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
u'message': u'success\n', u'type': u'status',
u'portal-type': u'promise', u'returncode': 0}
self.assertEquals(expected_result, result1)
# second run
promise_runner.runPromise()
result2 = json.loads(open(result_file).read().decode("utf-8"))
change_time2 = result2.pop('change-time', 2)
result2.pop('start-date')
change_date2 = datetime.fromtimestamp(change_time2)
self.assertTrue(result2.pop('execution-time', None) is not None)
self.assertEquals(expected_result, result2)
self.assertEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
change_date2.strftime('%Y-%m-%d %H:%M:%S'))
history_file = os.path.join(self.public_dir, 'promise_1.history.json')
self.assertTrue(os.path.exists(history_file))
history = json.load(open(history_file))
self.assertTrue(history['date'] > change_time)
self.assertTrue(len(history['data']) == 1)
def test_promise_generic_run_only(self):
promise_file = self.generatePromiseScript('my_promise.py', success=False)
second_promise = self.generatePromiseScript('my_second_promise.py', success=False)
parser = self.getPromiseParser()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result1['change-time'] = change_time
result1['start-date'] = start_date
result1['execution-time'] = execution_time
result1.pop('_links')
self.assertEquals(history['data'][0], result1)
result_file = os.path.join(self.output_dir, 'my_promise.status.json')
second_result = os.path.join(self.output_dir, 'my_second_promise.status.json')
self.assertTrue(os.path.exists(result_file))
self.assertTrue(os.path.exists(second_result))
def test_promise_NOK(self):
config = getArgumentParser().parse_args(['-c', self.monitor_config_file, '-a', '-f',
'--run-only', 'my_second_promise.py'])
os.unlink(result_file)
promise_runner = MonitorPromiseLauncher(config)
promise_runner.start()
self.assertFalse(os.path.exists(result_file))
self.assertTrue(os.path.exists(second_result))
promise = self.writePromiseNOK('promise_1')
def test_promise_OK(self):
promise = self.writePromiseOK('promise_1')
parser = self.getPromiseParser()
promise_runner = RunPromise(parser)
promise_runner.runPromise()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result_file = os.path.join(self.public_dir, 'promise_1.status.json')
result_file = os.path.join(self.output_dir, 'promise_1.status.json')
self.assertTrue(os.path.exists(result_file))
result1 = json.loads(open(result_file).read().decode("utf-8"))
change_time = result1.pop('change-time', 0)
change_date = datetime.fromtimestamp(change_time)
start_date = result1.pop('start-date')
self.assertTrue(result1.pop('execution-time', None) is not None)
expected_result = {u'status': u'ERROR', u'hosting_subscription': u'Monitor ROOT',
u'title': u'promise_1', u'instance': u'Monitor',
u'_links':
{u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
u'message': u'failed\n', u'type': u'status',
u'portal-type': u'promise', u'returncode': 2}
self.assertEquals(expected_result, result1)
start_date = datetime.strptime(result1['result'].pop('date'), '%Y-%m-%dT%H:%M:%S')
expected_result = {
u'title': u'promise_1', u'name': u'promise_1',
u'result': {
u'failed': False, u'message': u'success', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s' % promise,
}
# second run
promise_runner.runPromise()
parser = self.getPromiseParser(force=True)
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result2 = json.loads(open(result_file).read().decode("utf-8"))
change_time2 = result2.pop('change-time', 1)
result2.pop('start-date')
self.assertTrue(result2.pop('execution-time', None) is not None)
change_date2 = datetime.fromtimestamp(change_time2)
start_date2 = datetime.strptime(result2['result'].pop('date'), '%Y-%m-%dT%H:%M:%S')
self.assertEquals(expected_result, result2)
self.assertEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
change_date2.strftime('%Y-%m-%d %H:%M:%S'))
def test_promise_mixed(self):
def test_promise_two_folder(self):
promise = self.writePromiseOK('promise_1')
promise2 = self.writePromiseOK('promise_2')
parser = self.getPromiseParser()
promise_runner = RunPromise(parser)
promise_runner.runPromise()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result_file = os.path.join(self.public_dir, 'promise_1.status.json')
result_file = os.path.join(self.output_dir, 'promise_1.status.json')
result2_file = os.path.join(self.output_dir, 'promise_2.status.json')
self.assertTrue(os.path.exists(result_file))
self.assertTrue(os.path.exists(result2_file))
result1 = json.loads(open(result_file).read().decode("utf-8"))
change_time = result1.pop('change-time')
change_date = datetime.fromtimestamp(change_time)
start_date = result1.pop('start-date')
self.assertTrue(result1.pop('execution-time', None) is not None)
expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
u'title': u'promise_1', u'instance': u'Monitor',
u'_links':
{u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
u'message': u'success\n', u'type': u'status',
u'portal-type': u'promise', u'returncode': 0}
start_date = datetime.strptime(result1['result'].pop('date'), '%Y-%m-%dT%H:%M:%S')
expected_result = {
u'title': u'promise_1', u'name': u'promise_1',
u'result': {
u'failed': False, u'message': u'success', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s' % promise,
}
self.assertEquals(expected_result, result1)
# second run with failure
time.sleep(2)
promise = self.writePromiseNOK('promise_1')
parser = self.getPromiseParser()
expected_result['message'] = 'failed\n'
expected_result['status'] = 'ERROR'
expected_result['returncode'] = 2
promise_runner.runPromise()
result2 = json.loads(open(result_file).read().decode("utf-8"))
change_time2 = result2.pop('change-time')
result2.pop('start-date')
change_date2 = datetime.fromtimestamp(change_time2)
self.assertTrue(result2.pop('execution-time', None) is not None)
result2 = json.loads(open(result2_file).read())
start_date2 = datetime.strptime(result2['result'].pop('date'), '%Y-%m-%dT%H:%M:%S')
expected_result = {
u'title': u'promise_2', u'name': u'promise_2',
u'result': {
u'failed': False, u'message': u'success', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s' % promise2,
}
self.assertEquals(expected_result, result2)
self.assertNotEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
change_date2.strftime('%Y-%m-%d %H:%M:%S'))
def test_report_OK(self):
promise = self.writePromiseOK('sample_report')
parser = self.getUniquePromiseParser('sample_report', promise, 'report')
promise_runner = RunPromise(parser)
promise_runner.runPromise()
def test_promise_NOK(self):
promise = self.writePromiseNOK('promise_1')
parser = self.getPromiseParser()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result_file = os.path.join(self.private_dir, 'sample_report.report.json')
result_file = os.path.join(self.output_dir, 'promise_1.status.json')
self.assertTrue(os.path.exists(result_file))
result1 = json.loads(open(result_file).read().decode("utf-8"))
change_time = result1.pop('change-time', 0)
change_date = datetime.fromtimestamp(change_time)
start_date = result1.pop('start-date')
execution_time = result1.pop('execution-time')
expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
u'title': u'sample_report', u'instance': u'Monitor',
u'_links':
{u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
u'message': u'success\n', u'type': u'report',
u'portal-type': u'report', u'returncode': 0}
result1['result'].pop('date')
expected_result = {
u'title': u'promise_1', u'name': u'promise_1',
u'result': {
u'failed': True, u'message': u'failed', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s' % promise,
}
self.assertEquals(expected_result, result1)
# second run
promise_runner.runPromise()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result2 = json.loads(open(result_file).read().decode("utf-8"))
change_time2 = result2.pop('change-time', 2)
result2.pop('start-date')
result2.pop('execution-time')
change_date2 = datetime.fromtimestamp(change_time2)
result2['result'].pop('date')
self.assertEquals(expected_result, result2)
self.assertEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
change_date2.strftime('%Y-%m-%d %H:%M:%S'))
history_file = os.path.join(self.public_dir, 'sample_report.history.json')
self.assertTrue(os.path.exists(history_file))
history = json.load(open(history_file))
def test_promise_mixed(self):
promise = self.writePromiseOK('promise_1')
parser = self.getPromiseParser()
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result1['change-time'] = change_time
result1['start-date'] = start_date
result1['execution-time'] = execution_time
#result1.pop('_links')
self.assertEquals(history, result1)
def test_report_mixed(self):
promise = self.writePromiseOK('sample_report')
parser = self.getUniquePromiseParser('sample_report', promise, 'report')
promise_runner = RunPromise(parser)
promise_runner.runPromise()
result_file = os.path.join(self.private_dir, 'sample_report.report.json')
result_file = os.path.join(self.output_dir, 'promise_1.status.json')
self.assertTrue(os.path.exists(result_file))
result1 = json.loads(open(result_file).read().decode("utf-8"))
change_time = result1.pop('change-time', 0)
change_date = datetime.fromtimestamp(change_time)
start_date = result1.pop('start-date')
execution_time = result1.pop('execution-time')
expected_result = {u'status': u'OK', u'hosting_subscription': u'Monitor ROOT',
u'title': u'sample_report', u'instance': u'Monitor',
u'_links':
{u'monitor': {u'href': u'https://monitor.test.com/share/private/'}},
u'message': u'success\n', u'type': u'report',
u'portal-type': u'report', u'returncode': 0}
result1['result'].pop('date')
expected_result = {
u'title': u'promise_1', u'name': u'promise_1',
u'result': {
u'failed': False, u'message': u'success', u'type': u'Test Result'
},
u'execution-time': 0.1,
u'path': u'%s' % promise,
}
self.assertEquals(expected_result, result1)
# second run with failure
time.sleep(2)
promise = self.writePromiseNOK('sample_report')
parser = self.getUniquePromiseParser('sample_report', promise, 'report')
expected_result['message'] = 'failed\n'
expected_result['status'] = 'ERROR'
expected_result['returncode'] = 2
promise_runner.runPromise()
time.sleep(1)
promise = self.writePromiseNOK('promise_1')
parser = self.getPromiseParser(force=True)
expected_result['result']['message'] = u'failed'
expected_result['result']['failed'] = True
promise_runner = MonitorPromiseLauncher(parser)
promise_runner.start()
result2 = json.loads(open(result_file).read().decode("utf-8"))
change_time2 = result2.pop('change-time')
result2.pop('start-date')
result2.pop('execution-time')
change_date2 = datetime.fromtimestamp(change_time2)
result2['result'].pop('date')
self.assertEquals(expected_result, result2)
self.assertNotEquals(change_date.strftime('%Y-%m-%d %H:%M:%S'),
change_date2.strftime('%Y-%m-%d %H:%M:%S'))
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7