Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in / Register
Toggle navigation
S
slapos.core
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Labels
Merge Requests
16
Merge Requests
16
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Jobs
Commits
Open sidebar
nexedi
slapos.core
Commits
bbbaaeac
Commit
bbbaaeac
authored
Sep 10, 2014
by
Lutra Conseil
Committed by
Rafael Monnerat
Nov 10, 2014
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
slapos.collect: Compress historical logs and fix folder permissions.
parent
40246bbd
Changes
3
Show whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
63 additions
and
4 deletions
+63
-4
slapos/collect/__init__.py
slapos/collect/__init__.py
+10
-3
slapos/collect/reporter.py
slapos/collect/reporter.py
+26
-1
slapos/tests/collect.py
slapos/tests/collect.py
+27
-0
No files found.
slapos/collect/__init__.py
View file @
bbbaaeac
...
@@ -27,16 +27,17 @@
...
@@ -27,16 +27,17 @@
#
#
##############################################################################
##############################################################################
from
psutil
import
process_iter
,
NoSuchProcess
,
AccessDenied
from
psutil
import
process_iter
,
NoSuchProcess
,
AccessDenied
from
time
import
strftime
from
time
import
strftime
from
slapos.collect.db
import
Database
from
slapos.collect.db
import
Database
from
slapos.util
import
mkdir_p
from
slapos.util
import
mkdir_p
import
os
import
stat
from
slapos.collect.snapshot
import
ProcessSnapshot
,
ComputerSnapshot
from
slapos.collect.snapshot
import
ProcessSnapshot
,
ComputerSnapshot
from
slapos.collect.reporter
import
RawCSVDumper
,
\
from
slapos.collect.reporter
import
RawCSVDumper
,
\
SystemCSVReporterDumper
SystemCSVReporterDumper
,
\
compressLogFolder
from
entity
import
get_user_list
,
Computer
from
entity
import
get_user_list
,
Computer
...
@@ -80,6 +81,10 @@ def do_collect(conf):
...
@@ -80,6 +81,10 @@ def do_collect(conf):
log_directory
=
"%s/var/data-log"
%
conf
.
get
(
"slapos"
,
"instance_root"
)
log_directory
=
"%s/var/data-log"
%
conf
.
get
(
"slapos"
,
"instance_root"
)
mkdir_p
(
log_directory
,
0o755
)
mkdir_p
(
log_directory
,
0o755
)
if
stat
.
S_IMODE
(
os
.
stat
(
log_directory
).
st_mode
)
!=
0o755
:
os
.
chmod
(
log_directory
,
0o755
)
database
=
Database
(
log_directory
)
database
=
Database
(
log_directory
)
computer
=
Computer
(
ComputerSnapshot
())
computer
=
Computer
(
ComputerSnapshot
())
...
@@ -91,6 +96,8 @@ def do_collect(conf):
...
@@ -91,6 +96,8 @@ def do_collect(conf):
SystemCSVReporterDumper
(
database
).
dump
(
log_directory
)
SystemCSVReporterDumper
(
database
).
dump
(
log_directory
)
RawCSVDumper
(
database
).
dump
(
log_directory
)
RawCSVDumper
(
database
).
dump
(
log_directory
)
compressLogFolder
(
log_directory
)
# Drop older entries already reported
# Drop older entries already reported
database
.
garbageCollect
()
database
.
garbageCollect
()
...
...
slapos/collect/reporter.py
View file @
bbbaaeac
...
@@ -31,6 +31,9 @@
...
@@ -31,6 +31,9 @@
from
slapos.util
import
mkdir_p
from
slapos.util
import
mkdir_p
import
os.path
import
os.path
import
json
import
json
import
tarfile
import
glob
import
shutil
import
csv
import
csv
from
time
import
strftime
from
time
import
strftime
...
@@ -39,6 +42,12 @@ class Dumper(object):
...
@@ -39,6 +42,12 @@ class Dumper(object):
def
__init__
(
self
,
database
):
def
__init__
(
self
,
database
):
self
.
db
=
database
self
.
db
=
database
def
dump
(
self
,
folder
):
raise
NotImplemented
(
"Implemented on Subclass"
)
def
writeFile
(
self
,
**
kw
):
raise
NotImplemented
(
"Implemented on Subclass"
)
class
SystemReporter
(
Dumper
):
class
SystemReporter
(
Dumper
):
def
dump
(
self
,
folder
):
def
dump
(
self
,
folder
):
...
@@ -94,9 +103,25 @@ class RawDumper(Dumper):
...
@@ -94,9 +103,25 @@ class RawDumper(Dumper):
class
RawCSVDumper
(
RawDumper
):
class
RawCSVDumper
(
RawDumper
):
def
writeFile
(
self
,
name
,
folder
,
date_scope
,
rows
):
def
writeFile
(
self
,
name
,
folder
,
date_scope
,
rows
):
mkdir_p
(
os
.
path
.
join
(
folder
,
date_scope
))
mkdir_p
(
os
.
path
.
join
(
folder
,
date_scope
)
,
0o755
)
file_io
=
open
(
os
.
path
.
join
(
folder
,
"%s/dump_%s.csv"
%
(
date_scope
,
name
)),
"w"
)
file_io
=
open
(
os
.
path
.
join
(
folder
,
"%s/dump_%s.csv"
%
(
date_scope
,
name
)),
"w"
)
csv_output
=
csv
.
writer
(
file_io
)
csv_output
=
csv
.
writer
(
file_io
)
csv_output
.
writerows
(
rows
)
csv_output
.
writerows
(
rows
)
file_io
.
close
()
file_io
.
close
()
def
compressLogFolder
(
log_directory
):
initial_folder
=
os
.
getcwd
()
os
.
chdir
(
log_directory
)
try
:
for
backup_to_archive
in
glob
.
glob
(
"*-*-*/"
):
filename
=
'%s.tar.gz'
%
backup_to_archive
.
strip
(
"/"
)
with
tarfile
.
open
(
filename
,
'w:gz'
)
as
tfile
:
tfile
.
add
(
backup_to_archive
)
tfile
.
close
()
shutil
.
rmtree
(
backup_to_archive
)
finally
:
os
.
chdir
(
initial_folder
)
slapos/tests/collect.py
View file @
bbbaaeac
...
@@ -29,6 +29,7 @@ import os
...
@@ -29,6 +29,7 @@ import os
import
glob
import
glob
import
unittest
import
unittest
import
shutil
import
shutil
import
tarfile
import
tempfile
import
tempfile
import
slapos.slap
import
slapos.slap
import
psutil
import
psutil
...
@@ -290,6 +291,32 @@ class TestCollectReport(unittest.TestCase):
...
@@ -290,6 +291,32 @@ class TestCollectReport(unittest.TestCase):
self
.
assertEquals
(
set
(
glob
.
glob
(
"%s/*.csv"
%
self
.
instance_root
)),
set
(
csv_path_list
))
self
.
assertEquals
(
set
(
glob
.
glob
(
"%s/*.csv"
%
self
.
instance_root
)),
set
(
csv_path_list
))
def
test_compress_log_directory
(
self
):
log_directory
=
"%s/test_compress"
%
self
.
instance_root
dump_folder
=
"%s/1990-01-01"
%
log_directory
if
not
os
.
path
.
exists
(
log_directory
):
os
.
mkdir
(
log_directory
)
if
os
.
path
.
exists
(
dump_folder
):
shutil
.
rmtree
(
dump_folder
)
os
.
mkdir
(
"%s/1990-01-01"
%
log_directory
)
with
open
(
"%s/test.txt"
%
dump_folder
,
"w"
)
as
dump_file
:
dump_file
.
write
(
"hi"
)
dump_file
.
close
()
reporter
.
compressLogFolder
(
log_directory
)
self
.
assertFalse
(
os
.
path
.
exists
(
dump_folder
))
self
.
assertTrue
(
os
.
path
.
exists
(
"%s.tar.gz"
%
dump_folder
))
with
tarfile
.
open
(
"%s.tar.gz"
%
dump_folder
)
as
tf
:
self
.
assertEquals
(
tf
.
getmembers
()[
0
].
name
,
"1990-01-01"
)
self
.
assertEquals
(
tf
.
getmembers
()[
1
].
name
,
"1990-01-01/test.txt"
)
self
.
assertEquals
(
tf
.
extractfile
(
tf
.
getmembers
()[
1
]).
read
(),
'hi'
)
class
TestCollectSnapshot
(
unittest
.
TestCase
):
class
TestCollectSnapshot
(
unittest
.
TestCase
):
def
setUp
(
self
):
def
setUp
(
self
):
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment