Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
17
Merge Requests
17
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
slapos.core
Commits
1351084c
Commit
1351084c
authored
Oct 02, 2019
by
Rafael Monnerat
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
slapos/collect: Use decorator to open/close database
This is just to clean the methods a bit
parent
89250ec6
Changes
2
Hide whitespace changes
Inline
Side-by-side
Showing
2 changed files
with
24 additions
and
33 deletions
+24
-33
slapos/collect/db.py
slapos/collect/db.py
+1
-2
slapos/collect/reporter.py
slapos/collect/reporter.py
+23
-31
No files found.
slapos/collect/db.py
View file @
1351084c
...
@@ -169,13 +169,12 @@ class Database:
...
@@ -169,13 +169,12 @@ class Database:
return
strftime
(
"%Y-%m-d -- %H:%M:%S"
,
gmtime
()).
split
(
" -- "
)
return
strftime
(
"%Y-%m-d -- %H:%M:%S"
,
gmtime
()).
split
(
" -- "
)
def
has_table
(
self
,
name
):
def
has_table
(
self
,
name
):
self
.
connect
()
check_result_cursor
=
self
.
select
(
check_result_cursor
=
self
.
select
(
table
=
"sqlite_master"
,
table
=
"sqlite_master"
,
columns
=
'name'
,
columns
=
'name'
,
where
=
"type='table' AND name='%s'"
%
name
)
where
=
"type='table' AND name='%s'"
%
name
)
r
=
check_result_cursor
.
fetchone
()
r
=
check_result_cursor
.
fetchone
()
return
r
and
r
[
0
]
is
not
None
return
r
###################
###################
# Insertion methods
# Insertion methods
...
...
slapos/collect/reporter.py
View file @
1351084c
...
@@ -50,17 +50,26 @@ class Dumper(object):
...
@@ -50,17 +50,26 @@ class Dumper(object):
self
.
db
=
database
self
.
db
=
database
def
dump
(
self
,
folder
):
def
dump
(
self
,
folder
):
raise
NotImplemented
(
"Implemented on Subclass"
)
raise
NotImplemented
Error
(
"Implemented on Subclass"
)
def
writeFile
(
self
,
**
kw
):
def
writeFile
(
self
,
**
kw
):
raise
NotImplemented
(
"Implemented on Subclass"
)
raise
NotImplementedError
(
"Implemented on Subclass"
)
def
withDB
(
function
):
def
wrap_db_open_close
(
self
,
*
args
,
**
kwargs
):
self
.
db
.
connect
()
try
:
return
function
(
self
,
*
args
,
**
kwargs
)
finally
:
self
.
db
.
close
()
return
wraps
(
function
)(
wrap_db_open_close
)
class
SystemReporter
(
Dumper
):
class
SystemReporter
(
Dumper
):
@
withDB
def
dump
(
self
,
folder
):
def
dump
(
self
,
folder
):
""" Dump data """
""" Dump data """
_date
=
time
.
strftime
(
"%Y-%m-%d"
,
time
.
gmtime
())
_date
=
time
.
strftime
(
"%Y-%m-%d"
,
time
.
gmtime
())
self
.
db
.
connect
()
for
item
,
collected_item_list
in
six
.
iteritems
(
self
.
db
.
exportSystemAsDict
(
_date
)):
for
item
,
collected_item_list
in
six
.
iteritems
(
self
.
db
.
exportSystemAsDict
(
_date
)):
self
.
writeFile
(
item
,
folder
,
collected_item_list
)
self
.
writeFile
(
item
,
folder
,
collected_item_list
)
...
@@ -69,7 +78,6 @@ class SystemReporter(Dumper):
...
@@ -69,7 +78,6 @@ class SystemReporter(Dumper):
item
=
"memory_%s"
%
partition
.
split
(
"-"
)[
-
1
]
item
=
"memory_%s"
%
partition
.
split
(
"-"
)[
-
1
]
self
.
writeFile
(
"disk_%s_%s"
%
(
item
,
partition_id
),
folder
,
collected_item_list
)
self
.
writeFile
(
"disk_%s_%s"
%
(
item
,
partition_id
),
folder
,
collected_item_list
)
self
.
db
.
close
()
class
SystemJSONReporterDumper
(
SystemReporter
):
class
SystemJSONReporterDumper
(
SystemReporter
):
...
@@ -93,9 +101,9 @@ class SystemCSVReporterDumper(SystemReporter):
...
@@ -93,9 +101,9 @@ class SystemCSVReporterDumper(SystemReporter):
class
RawDumper
(
Dumper
):
class
RawDumper
(
Dumper
):
""" Dump raw data in a certain format
""" Dump raw data in a certain format
"""
"""
@
withDB
(
commit
=
1
)
def
dump
(
self
,
folder
):
def
dump
(
self
,
folder
):
date
=
time
.
strftime
(
"%Y-%m-%d"
,
time
.
gmtime
())
date
=
time
.
strftime
(
"%Y-%m-%d"
,
time
.
gmtime
())
self
.
db
.
connect
()
table_list
=
self
.
db
.
getTableList
()
table_list
=
self
.
db
.
getTableList
()
for
date_scope
,
amount
in
self
.
db
.
getDateScopeList
(
ignore_date
=
date
):
for
date_scope
,
amount
in
self
.
db
.
getDateScopeList
(
ignore_date
=
date
):
for
table
in
table_list
:
for
table
in
table_list
:
...
@@ -104,9 +112,6 @@ class RawDumper(Dumper):
...
@@ -104,9 +112,6 @@ class RawDumper(Dumper):
self
.
db
.
markDayAsReported
(
date_scope
,
self
.
db
.
markDayAsReported
(
date_scope
,
table_list
=
table_list
)
table_list
=
table_list
)
self
.
db
.
commit
()
self
.
db
.
close
()
class
RawCSVDumper
(
RawDumper
):
class
RawCSVDumper
(
RawDumper
):
def
writeFile
(
self
,
name
,
folder
,
date_scope
,
rows
):
def
writeFile
(
self
,
name
,
folder
,
date_scope
,
rows
):
...
@@ -134,8 +139,8 @@ class ConsumptionReportBase(object):
...
@@ -134,8 +139,8 @@ class ConsumptionReportBase(object):
def
__init__
(
self
,
db
):
def
__init__
(
self
,
db
):
self
.
db
=
db
self
.
db
=
db
@
withDB
def
getPartitionCPULoadAverage
(
self
,
partition_id
,
date_scope
):
def
getPartitionCPULoadAverage
(
self
,
partition_id
,
date_scope
):
self
.
db
.
connect
()
(
cpu_percent_sum
,),
=
self
.
db
.
select
(
"user"
,
date_scope
,
(
cpu_percent_sum
,),
=
self
.
db
.
select
(
"user"
,
date_scope
,
columns
=
"SUM(cpu_percent)"
,
columns
=
"SUM(cpu_percent)"
,
where
=
"partition = '%s'"
%
partition_id
)
where
=
"partition = '%s'"
%
partition_id
)
...
@@ -147,12 +152,10 @@ class ConsumptionReportBase(object):
...
@@ -147,12 +152,10 @@ class ConsumptionReportBase(object):
columns
=
"COUNT(DISTINCT time)"
,
columns
=
"COUNT(DISTINCT time)"
,
where
=
"partition = '%s'"
%
partition_id
)
where
=
"partition = '%s'"
%
partition_id
)
self
.
db
.
close
()
return
cpu_percent_sum
/
sample_amount
return
cpu_percent_sum
/
sample_amount
@
withDB
def
getPartitionUsedMemoryAverage
(
self
,
partition_id
,
date_scope
):
def
getPartitionUsedMemoryAverage
(
self
,
partition_id
,
date_scope
):
self
.
db
.
connect
()
(
memory_sum
,),
=
self
.
db
.
select
(
"user"
,
date_scope
,
(
memory_sum
,),
=
self
.
db
.
select
(
"user"
,
date_scope
,
columns
=
"SUM(memory_rss)"
,
columns
=
"SUM(memory_rss)"
,
where
=
"partition = '%s'"
%
partition_id
)
where
=
"partition = '%s'"
%
partition_id
)
...
@@ -164,12 +167,10 @@ class ConsumptionReportBase(object):
...
@@ -164,12 +167,10 @@ class ConsumptionReportBase(object):
columns
=
"COUNT(DISTINCT time)"
,
columns
=
"COUNT(DISTINCT time)"
,
where
=
"partition = '%s'"
%
partition_id
)
where
=
"partition = '%s'"
%
partition_id
)
self
.
db
.
close
()
return
memory_sum
/
sample_amount
return
memory_sum
/
sample_amount
@
withDB
def
getPartitionDiskUsedAverage
(
self
,
partition_id
,
date_scope
):
def
getPartitionDiskUsedAverage
(
self
,
partition_id
,
date_scope
):
self
.
db
.
connect
()
(
disk_used_sum
,),
=
self
.
db
.
select
(
"folder"
,
date_scope
,
(
disk_used_sum
,),
=
self
.
db
.
select
(
"folder"
,
date_scope
,
columns
=
"SUM(disk_used)"
,
columns
=
"SUM(disk_used)"
,
where
=
"partition = '%s'"
%
partition_id
)
where
=
"partition = '%s'"
%
partition_id
)
...
@@ -180,16 +181,14 @@ class ConsumptionReportBase(object):
...
@@ -180,16 +181,14 @@ class ConsumptionReportBase(object):
columns
=
"COUNT(DISTINCT time)"
,
columns
=
"COUNT(DISTINCT time)"
,
where
=
"partition = '%s'"
%
partition_id
)
where
=
"partition = '%s'"
%
partition_id
)
self
.
db
.
close
()
return
disk_used_sum
/
collect_amount
return
disk_used_sum
/
collect_amount
@
withDB
def
getPartitionProcessConsumptionList
(
self
,
partition_id
,
where
=
""
,
date_scope
=
None
,
def
getPartitionProcessConsumptionList
(
self
,
partition_id
,
where
=
""
,
date_scope
=
None
,
min_time
=
None
,
max_time
=
None
):
min_time
=
None
,
max_time
=
None
):
"""
"""
Query collector db to get consumed resource for last minute
Query collector db to get consumed resource for last minute
"""
"""
self
.
db
.
connect
()
consumption_list
=
[]
consumption_list
=
[]
if
where
!=
""
:
if
where
!=
""
:
where
=
"and %s"
%
where
where
=
"and %s"
%
where
...
@@ -232,12 +231,11 @@ class ConsumptionReportBase(object):
...
@@ -232,12 +231,11 @@ class ConsumptionReportBase(object):
resource_dict
[
'user'
]
=
pprocess
.
username
()
resource_dict
[
'user'
]
=
pprocess
.
username
()
resource_dict
[
'date'
]
=
datetime
.
fromtimestamp
(
pprocess
.
create_time
()).
strftime
(
"%Y-%m-%d %H:%M:%S"
)
resource_dict
[
'date'
]
=
datetime
.
fromtimestamp
(
pprocess
.
create_time
()).
strftime
(
"%Y-%m-%d %H:%M:%S"
)
consumption_list
.
append
(
resource_dict
)
consumption_list
.
append
(
resource_dict
)
self
.
db
.
close
()
return
consumption_list
return
consumption_list
@
withDB
def
getPartitionConsumptionStatusList
(
self
,
partition_id
,
where
=
""
,
def
getPartitionConsumptionStatusList
(
self
,
partition_id
,
where
=
""
,
date_scope
=
None
,
min_time
=
None
,
max_time
=
None
):
date_scope
=
None
,
min_time
=
None
,
max_time
=
None
):
self
.
db
.
connect
()
if
where
!=
""
:
if
where
!=
""
:
where
=
" and %s"
%
where
where
=
" and %s"
%
where
if
not
date_scope
:
if
not
date_scope
:
...
@@ -282,7 +280,6 @@ class ConsumptionReportBase(object):
...
@@ -282,7 +280,6 @@ class ConsumptionReportBase(object):
disk_used_sum
,
=
disk_result_cursor
.
fetchone
()
disk_used_sum
,
=
disk_result_cursor
.
fetchone
()
if
disk_used_sum
is
not
None
:
if
disk_used_sum
is
not
None
:
io_dict
[
'disk_used'
]
=
round
(
disk_used_sum
/
1024
,
2
)
io_dict
[
'disk_used'
]
=
round
(
disk_used_sum
/
1024
,
2
)
self
.
db
.
close
()
return
(
process_dict
,
memory_dict
,
io_dict
)
return
(
process_dict
,
memory_dict
,
io_dict
)
...
@@ -390,27 +387,22 @@ class ConsumptionReport(ConsumptionReportBase):
...
@@ -390,27 +387,22 @@ class ConsumptionReport(ConsumptionReportBase):
def
_getAverageFromList
(
self
,
data_list
):
def
_getAverageFromList
(
self
,
data_list
):
return
sum
(
data_list
)
/
len
(
data_list
)
return
sum
(
data_list
)
/
len
(
data_list
)
@
withDB
def
_getCpuLoadAverageConsumption
(
self
,
date_scope
):
def
_getCpuLoadAverageConsumption
(
self
,
date_scope
):
self
.
db
.
connect
()
(
cpu_load_percent_list
,),
=
self
.
db
.
select
(
"system"
,
date_scope
,
(
cpu_load_percent_list
,),
=
self
.
db
.
select
(
"system"
,
date_scope
,
columns
=
"SUM(cpu_percent)/COUNT(cpu_percent)"
)
columns
=
"SUM(cpu_percent)/COUNT(cpu_percent)"
)
self
.
db
.
close
()
return
cpu_load_percent_list
return
cpu_load_percent_list
@
withDB
def
_getMemoryAverageConsumption
(
self
,
date_scope
):
def
_getMemoryAverageConsumption
(
self
,
date_scope
):
self
.
db
.
connect
()
(
memory_used_list
,),
=
self
.
db
.
select
(
"system"
,
date_scope
,
(
memory_used_list
,),
=
self
.
db
.
select
(
"system"
,
date_scope
,
columns
=
"SUM(memory_used)/COUNT(memory_used)"
)
columns
=
"SUM(memory_used)/COUNT(memory_used)"
)
self
.
db
.
close
()
return
memory_used_list
return
memory_used_list
@
withDB
def
_getZeroEmissionContribution
(
self
):
def
_getZeroEmissionContribution
(
self
):
self
.
db
.
connect
()
return
self
.
db
.
getLastZeroEmissionRatio
()
zer
=
self
.
db
.
getLastZeroEmissionRatio
()
self
.
db
.
close
()
return
zer
class
Journal
(
object
):
class
Journal
(
object
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment