Skip to content

S
slapos.toolbox
Commit eabfaa8c authored by Cédric de Saint Martin's avatar Cédric de Saint Martin
Browse files
Options

Cleanup monitor.py

parent bb51f5da
Hide whitespace changes
Inline Side-by-side
Showing with 52 additions and 56 deletions
slapos/monitor.py
View file @ eabfaa8c
......@@ -28,15 +28,12 @@
import datetime
import os
import time
import resource
from time import strftime
import logging
import sys
from lxml import etree as ElementTree
import platform
import psutil
from optparse import OptionParser
import tempfile
import sqlite3.dbapi2 as sqlite3
#define global variable for log file
......@@ -48,27 +45,26 @@ class MonitoringTool(object):
pass
def get_cpu_and_memory_usage(self, proc):
"""Return CPU and Memory usage (percent) and
"""Return CPU and Memory usage (percent) and
the specific moment used for the measure"""
return [proc.get_cpu_percent(), sum(proc.get_cpu_times()), proc.get_memory_percent(), proc.get_memory_info()[0], datetime.datetime.now()]
class GenerateXML(object):
"""Return a XML file upon request by reading from database"""
def __init__(self, element_tree, path_database, path_xml):
self.element_tree = element_tree
self.path_database = path_database
self.path_xml = path_xml
#self.log_file = log_file
def dump_xml(self):
"""This func read data from database and through
_write_xml_output write result on xml file"""
_write_xml_output write result on xml file"""
consumption_infos = []
#This list hold the consuption infos in the following order
#[CPU % usage, CPU time usage (seconds), Memory % usage, Memory space usage (byte), date, time]
conn = sqlite3.connect(self.path_database)
cursor = conn.cursor()
cursor.execute("SELECT * FROM data")
......@@ -85,7 +81,7 @@ class GenerateXML(object):
conn.commit()
cursor.close()
conn.close()
def _eval_consumption_summary(self, consumption_infos):
"""This function return a resources usage summary, for pricing purpose"""
memory_percentage = []
......@@ -94,19 +90,19 @@ class GenerateXML(object):
total_cpu_time = 0.00
previous = 0.00
first_time = False
#The total time that the cpu spent to work on it
#Start-end time and date
start_time = consumption_infos[0][5]
end_time = consumption_infos[-1][5]
start_date = consumption_infos[0][4]
end_date = consumption_infos[-1][4]
for item in consumption_infos:
cpu_time.append(item[1])
memory_percentage.append(item[2])
memory_space.append(item[3])
#For all the samples, we calculate CPU consumption time
for indice,element in enumerate(cpu_time):
if indice == 0:
......@@ -127,8 +123,8 @@ class GenerateXML(object):
total_cpu_time = cpu_time[-1] - first_sample
#Else, we add the last sample to the total CPU consumption time
else:
total_cpu_time = total_cpu_time + cpu_time[-1]
total_cpu_time = total_cpu_time + cpu_time[-1]
return [total_cpu_time, sum(memory_space) / float(len(memory_space)), start_time, end_time, start_date, end_date]
#return [scipy.mean(cpu_percentage), cpu_time, scipy.mean(memory_percentage),
# scipy.mean(memory_space), start_time, end_time, start_date, end_date]
......@@ -136,26 +132,26 @@ class GenerateXML(object):
def _write_xml_output(self, res_list, storage_path):
"""This function provide to dump on xml the consumption infos,
"""This function provide to dump on xml the consumption infos,
the res_list contains the following informations:
[CPU mean %, CPU whole usage (seconds), Memory mean %, Memory mean space usage (byte),
[CPU mean %, CPU whole usage (seconds), Memory mean %, Memory mean space usage (byte),
start_time, end_time, start_date, end_date]"""
#XXX- NOTE
"""The res_list has been recently changed, now it contains
"""The res_list has been recently changed, now it contains
[CPU whole usage (seconds), Memory mean space usage (byte)]"""
res_list = map(str, res_list)
cpu_list = ['CPU Consumption',
'CPU consumption of the partition on %s at %s' % (res_list[5], res_list[3]),
res_list[0],
res_list[0],
]
memory_list = ['Memory consumption',
memory_list = ['Memory consumption',
'Memory consumption of the partition on %s at %s' % (res_list[5], res_list[3]),
res_list[1],
]
......@@ -170,10 +166,10 @@ class GenerateXML(object):
report = ElementTree.tostring(tree)
fd = open(storage_path, 'w')
fd.write("<?xml version='1.0' encoding='utf-8'?>%s" % report)
fd.close()
fd.close()
def _add_movement(self, consumption, single_resource_list):
child_consumption = ElementTree.SubElement(consumption, "movement")
child_movement = ElementTree.SubElement(child_consumption, "resource")
......@@ -208,16 +204,16 @@ Usage: slapreport [options] LOG_PATH DATABASE_PATH LOGBOX_IP LOGBOX_PORT LOGBOX_
return parser
class SlapMonitor(object):
def __init__(self, proc, update_time, path_database):
self.proc = proc
self.update_time = update_time
self.path_database = path_database
self.start_monitor()
self.start_monitor()
def start_monitor(self):
temporary_monitoring = MonitoringTool()
#check if the process is alive == None, instead zero value == proc is over
while self.proc.pid in psutil.get_pid_list():
......@@ -237,10 +233,10 @@ class SlapMonitor(object):
conn.close()
time.sleep(self.update_time)
except IOError:
if log_file:
logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
if log_file:
logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
% (self.proc.pid, strftime("%Y-%m-%d at %H:%m")))
sys.exit(1)
sys.exit(1)
if log_file:
logging.info("EXIT 0: Process terminated normally!")
sys.exit(0)
......@@ -248,7 +244,7 @@ class SlapMonitor(object):
class SlapReport(object):
"""
reports usage of apache and mariadb logs to an existing log server
reports usage of apache and mariadb logs to an existing log server
"""
def __init__(self, proc, update_time, consumption_log_path, database_path, ssh_parameters):
self.proc = proc
......@@ -256,7 +252,7 @@ class SlapReport(object):
self.path_database = database_path
self.consumption_log_path = consumption_log_path
self.ssh_parameters = ssh_parameters
self.start_report()
self.start_report()
def write_log(self):
"""This func read data from database and through
......@@ -270,23 +266,23 @@ class SlapReport(object):
last_report_time = ""
for row in log_list:
cpu_consumption_info = "%s %s Instance %s CPU Consumption: CPU:%s CPU_TIME:%s\n" \
% (row[4],row[5], self.proc.name, row[0],row[1])
% (row[4],row[5], self.proc.name, row[0],row[1])
memory_consumption_info = "%s %s Instance %s Memory Consumption: %s\n" \
% (row[4], row[5], self.proc.name, row[2])
% (row[4], row[5], self.proc.name, row[2])
try:
cmd = "tail -n 2 %s | ssh %s:%s@%s -p %s " \
% (self.consumption_log_path, self.ssh_parameters['user'], \
self.ssh_parameters['passwd'], self.ssh_parameters['ip'], \
self.ssh_parameters['port'])
cmd = "tail -n 2 %s | ssh %s:%s@%s -p %s " \
% (self.consumption_log_path, self.ssh_parameters['user'], \
self.ssh_parameters['passwd'], self.ssh_parameters['ip'], \
self.ssh_parameters['port'])
res = os.system(cmd)
if not res:
if not res:
if last_report_time != row[5]:
fd.write("%s%s" % (cpu_consumption_info,memory_consumption_info))
last_report_time = "%s" % row[5]
cursor.execute("UPDATE data set reported='1' WHERE time=?", (row[5],))
conn.commit()
except Exception, e:
conn.commit()
except Exception:
if log_file:
logging.info("ERROR : Unable to connect to % at %s"
% (self.ssh_parameters['ip'], strftime("%Y-%m-%d at %H:%m")))
......@@ -297,22 +293,22 @@ class SlapReport(object):
def start_report(self):
"""
while the process is running, connect to the database
while the process is running, connect to the database
and report the non-reported line,
when line is reported put 1 to column report
"""
temporary_monitoring = MonitoringTool()
#check if the process is alive == None, instead zero value == proc is over
while self.proc.pid in psutil.get_pid_list():
while self.proc.pid in psutil.get_pid_list():
try:
# send log to logbox
# send log to logbox
self.write_log()
time.sleep(self.update_time)
except IOError:
if log_file:
logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
if log_file:
logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
% (self.proc.pid, strftime("%Y-%m-%d at %H:%m")))
sys.exit(1)
sys.exit(1)
if log_file:
logging.info("EXIT 0: Process terminated normally!")
sys.exit(0)
......@@ -326,15 +322,15 @@ def run_slapmonitor():
if opts.path_log_file:
logging.basicConfig(filename=opts.path_log_file,level=logging.DEBUG)
global log_file
global log_file
log_file = True
fed = open(args[0], 'r')
pid_read = fed.read()
fed.close()
proc = psutil.Process(int(pid_read))
SlapMonitor(proc, opts.update_time, args[1])
def run_slapreport_():
#This function require the xml_path and database_path
parser = parse_opt()
......@@ -354,7 +350,7 @@ def run_slapreport():
#This function require the consumption_log_path and database_path ssh_parameters
parser = parse_opt()
ssh_parameters ={}
opts, args = parser.parse_args()
opts, args = parser.parse_args()
if len(args) != 7:
parser.error("Incorrect number of arguments, 7 required but "+str(len(args))+" detected" )
......@@ -363,9 +359,9 @@ def run_slapreport():
logging.basicConfig(filename=opts.path_log_file,level=logging.DEBUG)
global log_file
log_file = True
if args[3] == "":
sys.exit(0)
sys.exit(0)
pid_file_path = "%s" % args[0]
#set ssh parameters
......@@ -383,9 +379,9 @@ def run_slapreport():
except IOError:
if log_file:
logging.info("ERROR : process with pid : %s watched by slap monitor exited too quickly at %s"
% (self.proc.pid, strftime("%Y-%m-%d at %H:%m")))
% (proc.pid, strftime("%Y-%m-%d at %H:%m")))
sys.exit(1)
if log_file:
logging.info("EXIT 0: Process terminated normally!")
#sys.exit(0)
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7