Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
Kirill Smelkov
slapos
Commits
2e122eae
Commit
2e122eae
authored
Jan 30, 2015
by
Alain Takoudjou
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
monitor: show ressource consumption information
parent
4e0ba59a
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
390 additions
and
5 deletions
+390
-5
stack/monitor/buildout.cfg
stack/monitor/buildout.cfg
+20
-2
stack/monitor/collect.py
stack/monitor/collect.py
+163
-0
stack/monitor/monitor.cfg.in
stack/monitor/monitor.cfg.in
+14
-1
stack/monitor/webfile-directory/logfile.cgi.in
stack/monitor/webfile-directory/logfile.cgi.in
+3
-2
stack/monitor/webfile-directory/ressources.cgi.in
stack/monitor/webfile-directory/ressources.cgi.in
+190
-0
No files found.
stack/monitor/buildout.cfg
View file @
2e122eae
...
...
@@ -16,6 +16,8 @@ parts =
monitor-template
rss-bin
run-apachedex
collect-tools
log-tools
[monitor-eggs]
recipe = zc.recipe.egg
...
...
@@ -44,7 +46,7 @@ recipe = slapos.recipe.template
url = ${:_profile_base_location_}/monitor.cfg.in
output = ${buildout:directory}/monitor.cfg
filename = monitor.cfg
md5sum =
39c0b45d08399cf4a74fa586465fe8dc
md5sum =
05ed0063a8de43d3711b23605cdab4d7
mode = 0644
[monitor-bin]
...
...
@@ -94,10 +96,18 @@ mode = 0644
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfile-directory/${:filename}
download-only = true
md5sum =
223d8d98dd557ebc215be71f154629e7
md5sum =
d2b85ac31cbbf49c78dc51e90ddf305f
filename = logfile.cgi.in
mode = 0644
[ressources-cgi]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfile-directory/${:filename}
download-only = true
md5sum = 739a6e470ef174b9d0b523aa349860cb
filename = ressources.cgi.in
mode = 0644
[status-cgi]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/webfile-directory/${:filename}
...
...
@@ -159,6 +169,14 @@ md5sum = 7b38b3ab2d6c9ab657c8faf8e3f9b190
filename = logTools.py
mode = 0644
[collect-tools]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
download-only = true
md5sum = 3577a9bb13d49897a7e15422afdb11f4
filename = collect.py
mode = 0644
[errorlog-2rss]
recipe = hexagonit.recipe.download
url = ${:_profile_base_location_}/${:filename}
...
...
stack/monitor/collect.py
0 → 100644
View file @
2e122eae
# -*- coding: utf-8 -*-
##############################################################################
#
# Copyright (c) 2010-2014 Vifib SARL and Contributors.
# All Rights Reserved.
#
# WARNING: This program as such is intended to be used by professional
# programmers who take the whole responsibility of assessing all potential
# consequences resulting from its eventual inadequacies and bugs
# End users who are looking for a ready-to-use solution with commercial
# guarantees and support are strongly adviced to contract a Free Software
# Service Company
#
# This program is Free Software; you can redistribute it and/or
# modify it under the terms of the GNU Lesser General Public License
# as published by the Free Software Foundation; either version 2.1
# of the License, or (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU Lesser General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
#
##############################################################################
import
sqlite3
import
os
from
time
import
strftime
from
datetime
import
datetime
,
timedelta
class
Database
:
database_name
=
"collector.db"
table_list
=
[
"user"
,
"computer"
,
"system"
,
"disk"
,
\
"temperature"
,
"heating"
]
def
__init__
(
self
,
directory
=
None
):
assert
self
.
database_name
is
not
None
self
.
uri
=
os
.
path
.
join
(
directory
,
self
.
database_name
)
self
.
connection
=
None
self
.
cursor
=
None
def
connect
(
self
):
self
.
connection
=
sqlite3
.
connect
(
self
.
uri
)
self
.
cursor
=
self
.
connection
.
cursor
()
def
close
(
self
):
assert
self
.
connection
is
not
None
self
.
cursor
.
close
()
self
.
connection
.
close
()
def
_execute
(
self
,
sql
):
assert
self
.
connection
is
not
None
return
self
.
cursor
.
execute
(
sql
)
def
select
(
self
,
table
,
date
=
None
,
columns
=
"*"
,
where
=
None
):
""" Query database for a full table information """
if
date
is
not
None
:
where_clause
=
" WHERE date = '%s' "
%
date
else
:
where_clause
=
""
if
where
is
not
None
:
if
where_clause
==
""
:
where_clause
+=
" WHERE 1 = 1 "
where_clause
+=
" AND %s "
%
where
select_sql
=
"SELECT %s FROM %s %s "
%
(
columns
,
table
,
where_clause
)
return
self
.
_execute
(
select_sql
)
def
getPartitionCPULoadAverage
(
self
,
partition_id
,
date_scope
):
self
.
connect
()
query_result_cursor
=
self
.
select
(
"user"
,
date_scope
,
columns
=
"SUM(cpu_percent)"
,
where
=
"partition = '%s'"
%
partition_id
)
cpu_percent_sum
=
zip
(
*
query_result_cursor
)
if
len
(
cpu_percent_sum
)
and
cpu_percent_sum
[
0
][
0
]
is
None
:
return
query_result_cursor
=
self
.
select
(
"user"
,
date_scope
,
columns
=
"COUNT(DISTINCT time)"
,
where
=
"partition = '%s'"
%
partition_id
)
sample_amount
=
zip
(
*
query_result_cursor
)
self
.
close
()
if
len
(
sample_amount
)
and
len
(
cpu_percent_sum
):
return
round
(
cpu_percent_sum
[
0
][
0
]
/
sample_amount
[
0
][
0
],
2
)
def
getPartitionUsedMemoryAverage
(
self
,
partition_id
,
date_scope
):
self
.
connect
()
query_result_cursor
=
self
.
select
(
"user"
,
date_scope
,
columns
=
"SUM(memory_rss)"
,
where
=
"partition = '%s'"
%
partition_id
)
memory_sum
=
zip
(
*
query_result_cursor
)
if
len
(
memory_sum
)
and
memory_sum
[
0
][
0
]
is
None
:
return
query_result_cursor
=
self
.
select
(
"user"
,
date_scope
,
columns
=
"COUNT(DISTINCT time)"
,
where
=
"partition = '%s'"
%
partition_id
)
sample_amount
=
zip
(
*
query_result_cursor
)
self
.
close
()
if
len
(
sample_amount
)
and
len
(
memory_sum
):
return
round
(
memory_sum
[
0
][
0
]
/
sample_amount
[
0
][
0
],
2
)
def
getPartitionConsumption
(
self
,
partition_id
,
where
=
""
):
self
.
connect
()
comsumption_list
=
[]
if
where
!=
""
:
where
=
"and %s"
%
where
date_scope
=
datetime
.
now
().
strftime
(
'%Y-%m-%d'
)
min_time
=
(
datetime
.
now
()
-
timedelta
(
minutes
=
1
)).
strftime
(
'%H:%M:00'
)
max_time
=
(
datetime
.
now
()
-
timedelta
(
minutes
=
1
)).
strftime
(
'%H:%M:59'
)
sql_query
=
"""select count(pid), SUM(cpu_percent) as cpu_result, SUM(cpu_time),
MAX(cpu_num_threads), SUM(memory_percent), SUM(memory_rss), pid from user
where date='%s' and partition='%s' and (time between '%s' and '%s') %s
group by pid order by cpu_result desc"""
%
(
date_scope
,
partition_id
,
min_time
,
max_time
,
where
)
query_result
=
self
.
_execute
(
sql_query
)
for
result
in
query_result
:
count
=
int
(
result
[
0
])
if
not
count
>
0
:
continue
comsumption_list
.
append
([
result
[
6
],
round
((
result
[
1
]
/
count
),
2
),
round
((
result
[
2
]
/
count
),
2
),
round
(
result
[
3
],
2
),
round
((
result
[
4
]
/
count
),
2
),
round
((
result
[
5
]
/
count
),
2
)])
self
.
close
()
return
comsumption_list
def
getPartitionComsumptionStatus
(
self
,
partition_id
,
where
=
""
):
self
.
connect
()
if
where
!=
""
:
where
=
" and %s"
%
where
date_scope
=
datetime
.
now
().
strftime
(
'%Y-%m-%d'
)
min_time
=
(
datetime
.
now
()
-
timedelta
(
minutes
=
1
)).
strftime
(
'%H:%M:00'
)
max_time
=
(
datetime
.
now
()
-
timedelta
(
minutes
=
1
)).
strftime
(
'%H:%M:59'
)
sql_query
=
"""select count(pid), SUM(cpu_percent), SUM(cpu_time),
SUM(cpu_num_threads), SUM(memory_percent), SUM(memory_rss) from user where
date='%s' and partition='%s' and (time between '%s' and '%s') %s"""
%
(
date_scope
,
partition_id
,
min_time
,
max_time
,
where
)
query_result
=
self
.
_execute
(
sql_query
)
result_list
=
zip
(
*
query_result
)
self
.
close
()
if
len
(
result_list
):
result
=
result_list
#[0]
return
{
'total_process'
:
result
[
0
][
0
],
'cpu_percent'
:
round
(
result
[
1
][
0
],
2
),
'cpu_time'
:
round
(
result
[
2
][
0
],
2
),
'cpu_num_threads'
:
round
(
result
[
3
][
0
],
2
),
'memory_percent'
:
round
(
result
[
4
][
0
],
2
),
'memory_rss'
:
round
(
result
[
5
][
0
],
2
)}
return
None
stack/monitor/monitor.cfg.in
View file @
2e122eae
...
...
@@ -195,6 +195,19 @@ context =
[log-folder-cgi]
log-folder = $${monitor-directory:log}
[deploy-ressource-monitoring-cgi]
recipe = slapos.recipe.template:jinja2
template = ${ressources-cgi:location}/${ressources-cgi:filename}
rendered = $${monitor-directory:monitoring-cgi}/$${:filename}
filename = ressources.cgi
mode = 0744
# XXX - We need to find a proper way to set db_path here, maybe by using zero-parameters ??
context =
key monitor_bin monitor-parameters:executable
raw python_executable ${buildout:directory}/bin/${extra-eggs:interpreter}
key root_folder buildout:directory
raw db_path /srv/slapgrid/var/data-log/
[make-rss]
recipe = slapos.recipe.template:jinja2
template = ${make-rss-script:output}
...
...
@@ -370,7 +383,7 @@ certificate = $${ca-httpd:cert-file}
key = $${ca-httpd:key-file}
[httpd-environment]
PYTHONPATH = ${log-tools:location}
PYTHONPATH = ${log-tools:location}
:${collect-tools:location}
[monitor-httpd-configuration-file]
recipe = slapos.recipe.template:jinja2
...
...
stack/monitor/webfile-directory/logfile.cgi.in
View file @
2e122eae
...
...
@@ -201,7 +201,7 @@ else:
<input
type=
"hidden"
name=
"size"
id=
"size"
value=
"%s"
/>
</form>
<div
class=
"box"
>
<h2
class=
"head"
>
Tail
: %s
</h2>
<h2
class=
"head"
>
%s
: %s
</h2>
<div
class=
"button"
>
<button
type=
"submit"
class=
"pure-button pure-button-primary"
id=
"return"
>
Return
</button>
<button
type=
"submit"
class=
"pure-button pure-button-primary"
id=
"reload"
>
Refresh
</button>
...
...
@@ -209,7 +209,8 @@ else:
<div
style=
'clear:both'
></div>
<textarea
id=
"logcontent"
readonly
>
%s
</textarea>
</div>
""" % (script_path, logpath, action, pattern, size, title, log_content)
""" % (script_path, logpath, action, pattern, size, action.upper(),
title, log_content)
if pattern:
print "
<p>
Pattern string is: %s
</p>
" % pattern
print """
...
...
stack/monitor/webfile-directory/ressources.cgi.in
0 → 100644
View file @
2e122eae
#!{{ python_executable }}
import cgi
import cgitb
import json
import os
import pwd
from time import strftime
from datetime import datetime
import collect
cgitb.enable(display=0, logdir="/tmp/cgi.log")
form = cgi.FieldStorage()
db_path = "{{ db_path }}"
action = form.getvalue("action", "")
home = "{{ root_folder }}".strip()
if action:
db = collect.Database(directory=db_path)
stat_info = os.stat(home)
partition_user = pwd.getpwuid(stat_info.st_uid)[0]
result_dict = {}
date_scope = datetime.now().strftime('%Y-%m-%d')
if action == "refresh":
result_dict['consumption'] = db.getPartitionConsumption(partition_user)
result_dict['status'] = db.getPartitionComsumptionStatus(partition_user)
result_dict['cpu-load'] = db.getPartitionCPULoadAverage(
partition_user, date_scope)
result_dict['memory'] = db.getPartitionUsedMemoryAverage(
partition_user, date_scope)
print json.dumps(result_dict)
else:
print """
<html><head>
<link
rel=
"stylesheet"
href=
"static/pure-min.css"
>
<link
rel=
"stylesheet"
href=
"static/style.css"
>
<script
src=
"static/jquery-1.10.2.min.js"
></script>
<style
type=
"text/css"
>
.tg
{
border-collapse
:
collapse
;
border-spacing
:
0
;
border-color
:
#fff
;
border-width
:
1px
;
border-style
:
solid
;
width
:
100%
}
.tg
td
{
font-family
:
Arial
,
sans-serif
;
font-size
:
14px
;
padding
:
10px
5px
;
border-style
:
solid
;
border-width
:
0px
;
overflow
:
hidden
;
word-break
:
normal
;
border-color
:
#ccc
;
color
:
#333
;
background
:
transparent
;}
.tg
th
{
font-family
:
Arial
,
sans-serif
;
font-size
:
14px
;
font-weight
:
normal
;
padding
:
10px
5px
;
border-style
:
solid
;
border-width
:
0px
;
overflow
:
hidden
;
word-break
:
normal
;
border-color
:
#ccc
;
color
:
#333
;
background-color
:
#f0f0f0
;}
.tg
tr
{
background-color
:
#fff
;}
.tg
.tg-0ord
{
text-align
:
right
}
.tg
.tg-s6z2
td
{
text-align
:
center
}
.tg
.tg-zapm
{
background-color
:
#f9f9f9
;}
.tg
.tg-4eph
{
background-color
:
#f9f9f9
}
.head
{
background-color
:
#0078e7
;
border
:
0px
solid
#ffffff
;
text-align
:
left
;
border-width
:
0px
0px
1px
1px
;
font-size
:
18px
;
font-family
:
Helvetica
;
font-weight
:
normal
;
color
:
#ffffff
;
display
:
block
;
padding
:
10px
;
margin
:
0
;
}
.box
{
border
:
1px
solid
#e8eaed
;
padding
:
5px
;
}
</style>
<script
language=
"javascript"
type=
"text/javascript"
>
$
(
document
).
ready
(
function
()
{
var
send
=
false
;
autoRefresh
();
function
autoRefresh
()
{
refresh
();
setTimeout
(
function
(){
autoRefresh
();
},
60000
);
}
function
refresh
()
{
if
(
send
)
{
return
}
var
dataPost
=
{
'
posting-script
'
:
"
monitoring/ressources.cgi
"
,
action
:
"
refresh
"
};
send
=
true
;
$
(
"
#msg
"
).
fadeIn
();
$
.
ajax
({
type
:
"
POST
"
,
url
:
'
/index.cgi
'
,
data
:
dataPost
})
.
done
(
function
(
data
)
{
var
result
=
JSON
.
parse
(
data
);
var
consump
=
result
[
'
consumption
'
],
line
=
""
,
line2
=
""
;
var
table1
=
""
,
table1
=
""
,
table3
=
""
,
klass
=
""
;
table2
=
'
<tr><th class="tg-s6z2">Total Process</th>
'
;
table2
+=
'
<th class="tg-s6z2">CPU %</th><th class="tg-s6z2">CPU Time</th>
'
;
table2
+=
'
<th class="tg-s6z2">Threads</th><th class="tg-s6z2">Memory Usage</th>
'
;
table2
+=
'
<th class="tg-s6z2">Memory %</th></tr>
'
;
table1
=
'
<tr><th class="tg-s6z2">Process PID</th>
'
;
table1
+=
'
<th class="tg-s6z2">CPU %</th><th class="tg-s6z2">CPU Time</th>
'
;
table1
+=
'
<th class="tg-s6z2">Threads</th><th class="tg-s6z2">Memory Usage</th>
'
;
table1
+=
'
<th class="tg-s6z2">Memory %</th></tr>
'
;
table3
=
'
<tr><th class="tg-s6z2">CPU Load Average</th>
'
;
table3
+=
'
<th class="tg-s6z2">Memory Consumption Average</th></tr>
'
;
line2
=
"
<tr class='tg-4eph tg-s6z2'>
"
line2
+=
"
<td>
"
+
result
[
'
status
'
][
'
total_process
'
]
+
"
</td>
"
;
line2
+=
"
<td>
"
+
result
[
'
status
'
][
'
cpu_percent
'
]
+
"
</td>
"
;
line2
+=
"
<td>
"
+
result
[
'
status
'
][
'
cpu_time
'
]
+
"
</td>
"
;
line2
+=
"
<td>
"
+
result
[
'
status
'
][
'
cpu_num_threads
'
]
+
"
</td>
"
;
line2
+=
"
<td>
"
+
result
[
'
status
'
][
'
memory_rss
'
]
+
"
</td>
"
;
line2
+=
"
<td>
"
+
result
[
'
status
'
][
'
memory_percent
'
]
+
"
</td>
"
;
line2
+=
"
</tr>
"
;
for
(
var
i
=
0
;
i
<
consump
.
length
;
i
++
)
{
if
(
klass
===
""
)
{
klass
=
'
tg-4eph
'
;}
else
{
klass
=
""
;}
line
+=
"
<tr class='
"
+
klass
+
"
tg-s6z2'>
"
line
+=
"
<td>
"
+
consump
[
i
][
0
]
+
"
</td>
"
;
line
+=
"
<td>
"
+
consump
[
i
][
1
]
+
"
</td>
"
;
line
+=
"
<td>
"
+
consump
[
i
][
2
]
+
"
</td>
"
;
line
+=
"
<td>
"
+
consump
[
i
][
3
]
+
"
</td>
"
;
line
+=
"
<td>
"
+
consump
[
i
][
5
]
+
"
</td>
"
;
line
+=
"
<td>
"
+
consump
[
i
][
4
]
+
"
</td>
"
;
line
+=
"
</tr>
"
;
}
table3
+=
"
<tr class='tg-4eph tg-s6z2'>
"
table3
+=
"
<td>
"
+
result
[
'
cpu-load
'
]
+
"
</td>
"
;
table3
+=
"
<td>
"
+
result
[
'
memory
'
]
+
"
</td></tr></table>
"
;
$
(
"
#box3
"
).
html
(
table3
);
$
(
"
#box2
"
).
html
(
table2
+
line2
+
'
</table>
'
);
$
(
"
#box1
"
).
html
(
table1
+
line
+
'
</table>
'
);
})
.
fail
(
function
(
jqXHR
,
exception
)
{
$
(
"
#error
"
).
html
(
jqXHR
);
})
.
always
(
function
()
{
send
=
false
;
$
(
"
#msg
"
).
fadeOut
();
});
}
$
(
"
#refresh
"
).
click
(
function
()
{
refresh
();
});
});
</script>
</head><body>
<h1>
Computer partition ressources monitoring
</h1>
<div
style=
"width:850px; padding: 10px 0;"
>
<div
style=
'float:left; width: 500px'
>
<table
class=
"tg"
id=
"box3"
>
<tr>
<th
class=
"tg-s6z2"
>
CPU Load Average
</th>
<th
class=
"tg-s6z2"
>
Memory Consumption Average
</th>
</tr>
</table>
</div>
<div
style=
'float:left; padding-left:50px;'
>
<button
type=
"button"
class=
"pure-button pure-button-primary"
id=
"refresh"
>
Refresh data
</button>
<span
style=
"padding: 5px; color:rgb(217, 39, 39); display:none"
id=
"msg"
>
Loading data...
</span>
<br/>
<p>
Note: Data are refreshed every minutes.
</p>
</div>
</div>
<div
style=
'clear:both'
></div>
<h2>
Total ressources consumption for partition (last minute)
</h2>
<table
class=
"tg"
id=
"box2"
>
<tr>
<th
class=
"tg-s6z2"
>
Total Process
</th>
<th
class=
"tg-s6z2"
>
CPU %
</th>
<th
class=
"tg-s6z2"
>
CPU Time
</th>
<th
class=
"tg-s6z2"
>
Threads
</th>
<th
class=
"tg-s6z2"
>
Memory Usage
</th>
<th
class=
"tg-s6z2"
>
Memory %
</th>
</tr>
</table>
<h2>
ressources consumption for partition by process pid (last minute)
</h2>
<table
class=
"tg"
id=
"box1"
>
<tr>
<th
class=
"tg-s6z2"
>
Process PID
</th>
<th
class=
"tg-s6z2"
>
CPU %
</th>
<th
class=
"tg-s6z2"
>
CPU Time
</th>
<th
class=
"tg-s6z2"
>
Threads
</th>
<th
class=
"tg-s6z2"
>
Memory Usage
</th>
<th
class=
"tg-s6z2"
>
Memory %
</th>
</tr>
</table>
<p
id=
"error"
></p>
</body></html>
"""
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment