Skip to content

M
metadata-collect-agent
Commit 60fb5186 authored by Ophélie Gagnard's avatar Ophélie Gagnard
Browse files
Options

Improve cython's Makefile. Remove useless files.

parent 5fc7f057
Hide whitespace changes
Inline Side-by-side
Showing with 5 additions and 1014 deletions
scan-filesystem/cython/Makefile
View file @ 60fb5186
ifeq (,) ifeq (,)
INCLUDE_DIRS = -I/usr/include/python3.7 INCLUDE_PYTHON = $(shell pkg-config --cflags python3 | cut -f 1 -d " ")
else else
...@@ -34,25 +34,14 @@ LDLIBS = -lcrypto -lfmt ...@@ -34,25 +34,14 @@ LDLIBS = -lcrypto -lfmt
EXT_SUFFIX := $(shell python3 -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))") EXT_SUFFIX := $(shell python3 -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
EXT = $(EXE)$(EXT_SUFFIX) EXT = $(EXE)$(EXT_SUFFIX)
# Build with Python runtime
all: $(EXT)
$(EXT): setup.py
@echo "[Cython Compiling $^ -> $@]"
python3 setup.py build_ext --inplace
# Run with Python runtime
run: $(EXT)
python3 -c "import $(EXE); $(EXE).python_main()" 2>/dev/null
# Build without Python runtime # Build without Python runtime
nopython: main.cpp#$(EXE) all: main.cpp#$(EXE)
mkdir -p logs mkdir -p logs
# Error expected here, writing a C program using the error messages # Error expected here, writing a C program using the error messages
-g++ -O2 -g -Wno-unused-result -Wsign-compare -pthread -I/opt/python-3.7.12/include/python3.7m/ main.cpp -lcrypto -lfmt -o main 2> logs/link_errors -g++ -O2 -g -Wno-unused-result -Wsign-compare -pthread $(INCLUDE_PYTHON) main.cpp -lcrypto -lfmt -o main 2> logs/link_errors
./parse_link_errors.py < logs/link_errors ./parse_link_errors.py < logs/link_errors
make fake_python.o make fake_python.o
g++ -O2 -g -Wno-unused-result -Wsign-compare -pthread -I/opt/python-3.7.12/include/python3.7m/ main.cpp fake_python.o -lcrypto -lfmt -o metadata-collect-agent g++ -O2 -g -Wno-unused-result -Wsign-compare -pthread $(INCLUDE_PYTHON) main.cpp fake_python.o -lcrypto -lfmt -o metadata-collect-agent
#-ln -f ../../metadata-collect-agent metadata-collect-agent #-ln -f ../../metadata-collect-agent metadata-collect-agent
-ln -f metadata-collect-agent ../../dracut.module/90metadata-collect/metadata-collect-agent -ln -f metadata-collect-agent ../../dracut.module/90metadata-collect/metadata-collect-agent
#-ln -f metadata-collect-agent ../../debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent #-ln -f metadata-collect-agent ../../debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent
...@@ -66,10 +55,6 @@ nopython: main.cpp#$(EXE) ...@@ -66,10 +55,6 @@ nopython: main.cpp#$(EXE)
@echo "[C++ Compiling $^ -> $@]" @echo "[C++ Compiling $^ -> $@]"
$(LINK.cpp) $^ $(LOADLIBES) $(LDLIBS) -o $@ $(LINK.cpp) $^ $(LOADLIBES) $(LDLIBS) -o $@
# Run without Python runtime
runnopython: $(EXE)
./$(EXE) 2>/dev/null
clean: clean:
-rm -f *.c *.cpp *.html -rm -f *.c *.cpp *.html
-rm -f *.h -rm -f *.h
...@@ -84,5 +69,5 @@ clean: ...@@ -84,5 +69,5 @@ clean:
-rm -f ../../debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent -rm -f ../../debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent
-rm -rf logs -rm -rf logs
.PHONY: all run nopython runnopython clean .PHONY: all clean
.PRECIOUS: %.cpp .PRECIOUS: %.cpp
scan-filesystem/cython/command-line.main.pyx deleted 100644 → 0
View file @ 5fc7f057
# distutils: language = c++
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
from runtime.wait cimport wait, waitpid
from stdlib.stat cimport Stat, dev_t
from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.stdlib cimport realpath
from posix.unistd cimport readlink
cdef lock Scheduler scheduler
cdef cypclass Node activable:
string path
string name
Stat st
string formatted
__init__(self, string path, string name, Stat st):
self._active_result_class = NullResult
self._active_queue_class = consume BatchMailBox(scheduler)
self.path = path
self.name = name
self.st = st
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
# abstract
pass
void format_node(self):
self.formatted = sprintf("""{"path": "%s", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
# abstract
pass
cdef iso Node make_node(string path, string name) nogil:
s = Stat(path)
if s is NULL:
return NULL
elif s.is_symlink():
return consume SymlinkNode(path, name, consume s)
elif s.is_dir():
return consume DirNode(path, name, consume s)
elif s.is_regular():
return consume FileNode(path, name, consume s)
return NULL
cdef cypclass DirNode(Node):
cyplist[active Node] children
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.children = new cyplist[active Node]()
self.children.__init__()
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef DIR *d
cdef struct_dirent *entry
cdef string entry_name
cdef string entry_path
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
#""" temporarily desabling white list
elif not self.st.st_data.st_dev in dev_whitelist:
return
#"""
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
entry = readdir(d)
if entry is NULL:
break
entry_name = entry.d_name
if entry_name == b'.' or entry_name == b'..':
continue
entry_path = self.path
if entry_path != b'/':
entry_path += b'/'
entry_path += entry_name
entry_node = make_node(entry_path, entry_name)
if entry_node is NULL:
continue
active_entry = activate(consume entry_node)
self.children.append(active_entry)
closedir(d)
self.format_node()
for active_child in self.children:
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void format_node(self):
self.formatted = sprintf("""{"path": "%s/", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
while self.children.__len__() > 0:
active_child = self.children[self.children.__len__() -1]
del self.children[self.children.__len__() -1]
child = consume active_child
child.write_node(stream)
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
string sha256_data
string sha512_data
bint error
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.error = False
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef unsigned char buffer[BUFSIZE]
cdef bint eof = False
cdef bint md5_ok
cdef bint sha1_ok
cdef bint sha256_ok
cdef bint sha512_ok
cdef FILE * file = fopen(self.path.c_str(), 'rb')
if file is NULL:
self.error = True
self.format_node()
return
md5 = MessageDigest(md5sum())
sha1 = MessageDigest(sha1sum())
sha256 = MessageDigest(sha256sum())
sha512 = MessageDigest(sha512sum())
md5_ok = md5 is not NULL
sha1_ok = sha1 is not NULL
sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL
while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
size = fread(buffer, 1, BUFSIZE, file)
if size != BUFSIZE:
self.error = ferror(file)
if self.error:
break
eof = True
if md5_ok: md5_ok = md5.update(buffer, size) == 0
if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
fclose(file)
if not self.error:
if md5_ok: self.md5_data = md5.hexdigest()
if sha1_ok: self.sha1_data = sha1.hexdigest()
if sha256_ok: self.sha256_data = sha256.hexdigest()
if sha512_ok: self.sha512_data = sha512.hexdigest()
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""{"path": "%s", "stat": %s, "hash": {"md5": "%s", "sha1": "%s", "sha256": "%s", "sha512": "%s"}}\n""",
self.path,
self.st.to_json(),
self.md5_data,
self.sha1_data,
self.sha256_data,
self.sha512_data,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef cypclass SymlinkNode(Node):
string target
int error
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
size = self.st.st_data.st_size + 1
self.target.resize(size)
real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
self.error = not (0 < real_size < size)
self.target.resize(real_size)
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""{"path": "%s", "stat": %s, "target": "%s"}\n""",
self.path,
self.st.to_json(),
self.target,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(const char *path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
# TODO replace 4096 by PATH_MAX (yet it will not be perfect)
cdef char resolved_path[4096]
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "flb/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "flb/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
#cdef char ip_address[100]
#cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("flb/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
printf("WELCOME TO PARENT\n\n") # DEBUG
sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
global scheduler
scheduler = Scheduler()
ignore_paths = cyplist[string]()
ignore_paths.append(b'/opt/slapgrid')
ignore_paths.append(b'/srv/slapgrid')
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot/efi')
dev_whitelist_paths.append(b'/root')
dev_whitelist_paths.append(b'/sysroot')
dev_whitelist_paths.append(path)
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
p_stat = Stat(p)
if p_stat is not NULL:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
realpath(path, resolved_path)
fprintf(stderr, resolved_path) # DEBUG
fprintf(stderr, "\n") # DEBUG
fflush(stderr) # DEBUG
node = make_node(resolved_path, resolved_path)
if node is NULL:
return -1
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
scheduler.finish()
node = consume active_node
#""" # DEBUG
result = fopen('flb/metadata_collect.log', 'w')
if result is NULL:
fprintf(stderr, 'Error creating the log file.\n') # DEBUG
fflush(stderr)
return -1
fprintf(stderr, 'Log opened successfully.\n') # DEBUG
fprintf(stderr, "WRITE_NOTE STAGE\n\n") # DEBUG
#fscanf(address_path, "%s", ip_address)
#fclose(address_path)
#fprintf(result, '{"mac_address": "%s"}\n', ip_address)
node.write_node(result)
fprintf(result, '{}\n')
fprintf(result, 'fluentbit_end\n')
fclose(result)
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS\n\n")
fflush(stderr) # DEBUG
return 0
cdef public int main(int argc, char* argv[]) nogil:
if argc >= 2:
return start(<char*>argv[1])
else:
return start(<char*>'/')
#def python_main():
# start(<char*>'.')
scan-filesystem/cython/new.uefi.main.pyx deleted 100644 → 0
View file @ 5fc7f057
# distutils: language = c++
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
from runtime.wait cimport wait, waitpid
from stdlib.stat cimport Stat, dev_t
from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.stdlib cimport realpath
from posix.unistd cimport readlink
cdef lock Scheduler scheduler
cdef cypclass Node activable:
string path
string name
Stat st
string formatted
__init__(self, string path, string name, Stat st):
self._active_result_class = NullResult
self._active_queue_class = consume BatchMailBox(scheduler)
self.path = path
self.name = name
self.st = st
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
# abstract
pass
void format_node(self):
self.formatted = sprintf("""{"path": "%s", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
# abstract
pass
cdef iso Node make_node(string path, string name) nogil:
s = Stat(path)
if s is NULL:
return NULL
elif s.is_symlink():
return consume SymlinkNode(path, name, consume s)
elif s.is_dir():
return consume DirNode(path, name, consume s)
elif s.is_regular():
return consume FileNode(path, name, consume s)
return NULL
cdef cypclass DirNode(Node):
cyplist[active Node] children
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.children = new cyplist[active Node]()
self.children.__init__()
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef DIR *d
cdef struct_dirent *entry
cdef string entry_name
cdef string entry_path
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
""" temporarily desabling white list
elif not self.st.st_data.st_dev in dev_whitelist:
return
"""
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
entry = readdir(d)
if entry is NULL:
break
entry_name = entry.d_name
if entry_name == b'.' or entry_name == b'..':
continue
entry_path = self.path
if entry_path != b'/':
entry_path += b'/'
entry_path += entry_name
entry_node = make_node(entry_path, entry_name)
if entry_node is NULL:
continue
active_entry = activate(consume entry_node)
self.children.append(active_entry)
closedir(d)
self.format_node()
for active_child in self.children:
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void format_node(self):
self.formatted = sprintf("""{"path": "%s/", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
while self.children.__len__() > 0:
active_child = self.children[self.children.__len__() -1]
del self.children[self.children.__len__() -1]
child = consume active_child
child.write_node(stream)
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
string sha256_data
string sha512_data
bint error
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.error = False
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef unsigned char buffer[BUFSIZE]
cdef bint eof = False
cdef bint md5_ok
cdef bint sha1_ok
cdef bint sha256_ok
cdef bint sha512_ok
cdef FILE * file = fopen(self.path.c_str(), 'rb')
if file is NULL:
self.error = True
self.format_node()
return
md5 = MessageDigest(md5sum())
sha1 = MessageDigest(sha1sum())
sha256 = MessageDigest(sha256sum())
sha512 = MessageDigest(sha512sum())
md5_ok = md5 is not NULL
sha1_ok = sha1 is not NULL
sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL
while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
size = fread(buffer, 1, BUFSIZE, file)
if size != BUFSIZE:
self.error = ferror(file)
if self.error:
break
eof = True
if md5_ok: md5_ok = md5.update(buffer, size) == 0
if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
fclose(file)
if not self.error:
if md5_ok: self.md5_data = md5.hexdigest()
if sha1_ok: self.sha1_data = sha1.hexdigest()
if sha256_ok: self.sha256_data = sha256.hexdigest()
if sha512_ok: self.sha512_data = sha512.hexdigest()
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""{"path": "%s", "stat": %s, "hash": {"md5": "%s", "sha1": "%s", "sha256": "%s", "sha512": "%s"}}\n""",
self.path,
self.st.to_json(),
self.md5_data,
self.sha1_data,
self.sha256_data,
self.sha512_data,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef cypclass SymlinkNode(Node):
string target
int error
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
size = self.st.st_data.st_size + 1
self.target.resize(size)
real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
self.error = not (0 < real_size < size)
self.target.resize(real_size)
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""{"path": "%s", "stat": %s, "target": "%s"}\n""",
self.path,
self.st.to_json(),
self.target,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(const char *path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
# TODO replace 4096 by PATH_MAX (yet it will not be perfect)
cdef char resolved_path[4096]
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "/etc/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "/etc/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
#cdef char ip_address[100]
#cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("/sbin/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
printf("WELCOME TO PARENT\n\n") # DEBUG
sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
global scheduler
scheduler = Scheduler()
ignore_paths = cyplist[string]()
ignore_paths.append(b'/opt/slapgrid')
ignore_paths.append(b'/srv/slapgrid')
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot/efi')
dev_whitelist_paths.append(b'/root')
dev_whitelist_paths.append(b'/sysroot')
dev_whitelist_paths.append(path)
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
p_stat = Stat(p)
if p_stat is not NULL:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
realpath(path, resolved_path)
fprintf(stderr, resolved_path) # DEBUG
fprintf(stderr, "\n") # DEBUG
fflush(stderr) # DEBUG
node = make_node(resolved_path, resolved_path)
if node is NULL:
return -1
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
scheduler.finish()
node = consume active_node
#""" # DEBUG
result = fopen('/var/log/metadata_collect.log', 'w')
if result is NULL:
fprintf(stderr, 'Error creating the log file.\n') # DEBUG
fflush(stderr)
return -1
fprintf(stderr, 'Log opened successfully.\n') # DEBUG
fprintf(stderr, "WRITE_NOTE STAGE\n\n") # DEBUG
#fscanf(address_path, "%s", ip_address)
#fclose(address_path)
#fprintf(result, '{"mac_address": "%s"}\n', ip_address)
node.write_node(result)
fprintf(result, '{}\n')
fprintf(result, 'fluentbit_end\n')
fclose(result)
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS\n\n")
fflush(stderr) # DEBUG
return 0
cdef public int main(int argc, char* argv[]) nogil:
if argc >= 2:
return start(<char*>argv[1])
else:
return start(<char*>'/')
#def python_main():
# start(<char*>'.')
scan-filesystem/cython/uefi.main.pyx deleted 100644 → 0
View file @ 5fc7f057
# distutils: language = c++
from libcythonplus.list cimport cyplist
from libc.stdio cimport fprintf, fopen, fclose, fread, fwrite, FILE, stdout, stderr, printf, ferror, fscanf, fflush
from runtime.runtime cimport SequentialMailBox, BatchMailBox, NullResult, Scheduler
from runtime.unistd cimport pid_t, execlp, fork, sleep
from runtime.wait cimport wait, waitpid
from stdlib.stat cimport Stat, dev_t
from stdlib.digest cimport MessageDigest, md5sum, sha1sum, sha256sum, sha512sum
from stdlib.fmt cimport sprintf
from stdlib.string cimport string
from stdlib.dirent cimport DIR, struct_dirent, opendir, readdir, closedir
from posix.stdlib cimport realpath
from posix.unistd cimport readlink
cdef lock Scheduler scheduler
cdef cypclass Node activable:
string path
string name
Stat st
string formatted
__init__(self, string path, string name, Stat st):
self._active_result_class = NullResult
self._active_queue_class = consume BatchMailBox(scheduler)
self.path = path
self.name = name
self.st = st
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
# abstract
pass
void format_node(self):
self.formatted = sprintf("""{"path": "%s", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
# abstract
pass
cdef iso Node make_node(string path, string name) nogil:
s = Stat(path)
if s is NULL:
return NULL
elif s.is_symlink():
return consume SymlinkNode(path, name, consume s)
elif s.is_dir():
return consume DirNode(path, name, consume s)
elif s.is_regular():
return consume FileNode(path, name, consume s)
return NULL
cdef cypclass DirNode(Node):
cyplist[active Node] children
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.children = new cyplist[active Node]()
self.children.__init__()
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef DIR *d
cdef struct_dirent *entry
cdef string entry_name
cdef string entry_path
if ignore_paths is not NULL:
if self.path in ignore_paths:
return
if dev_whitelist is not NULL:
if self.st is NULL:
return
#""" temporarily desabling white list
elif not self.st.st_data.st_dev in dev_whitelist:
return
#"""
d = opendir(self.path.c_str())
if d is not NULL:
while 1:
entry = readdir(d)
if entry is NULL:
break
entry_name = entry.d_name
if entry_name == b'.' or entry_name == b'..':
continue
entry_path = self.path
if entry_path != b'/':
entry_path += b'/'
entry_path += entry_name
entry_node = make_node(entry_path, entry_name)
if entry_node is NULL:
continue
active_entry = activate(consume entry_node)
self.children.append(active_entry)
closedir(d)
self.format_node()
for active_child in self.children:
active_child.build_node(NULL, dev_whitelist, ignore_paths)
void format_node(self):
self.formatted = sprintf("""{"path": "%s/", "stat": %s}\n""",
self.path,
self.st.to_json(),
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
while self.children.__len__() > 0:
active_child = self.children[self.children.__len__() -1]
del self.children[self.children.__len__() -1]
child = consume active_child
child.write_node(stream)
cdef enum:
BUFSIZE = 64 * 1024
cdef cypclass FileNode(Node):
string md5_data
string sha1_data
string sha256_data
string sha512_data
bint error
__init__(self, string path, string name, Stat st):
Node.__init__(self, path, name, st)
self.error = False
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
cdef unsigned char buffer[BUFSIZE]
cdef bint eof = False
cdef bint md5_ok
cdef bint sha1_ok
cdef bint sha256_ok
cdef bint sha512_ok
cdef FILE * file = fopen(self.path.c_str(), 'rb')
if file is NULL:
self.error = True
self.format_node()
return
md5 = MessageDigest(md5sum())
sha1 = MessageDigest(sha1sum())
sha256 = MessageDigest(sha256sum())
sha512 = MessageDigest(sha512sum())
md5_ok = md5 is not NULL
sha1_ok = sha1 is not NULL
sha256_ok = sha256 is not NULL
sha512_ok = sha512 is not NULL
while not eof and (md5_ok or sha1_ok or sha256_ok or sha512_ok):
size = fread(buffer, 1, BUFSIZE, file)
if size != BUFSIZE:
self.error = ferror(file)
if self.error:
break
eof = True
if md5_ok: md5_ok = md5.update(buffer, size) == 0
if sha1_ok: sha1_ok = sha1.update(buffer, size) == 0
if sha256_ok: sha256_ok = sha256.update(buffer, size) == 0
if sha512_ok: sha512_ok = sha512.update(buffer, size) == 0
fclose(file)
if not self.error:
if md5_ok: self.md5_data = md5.hexdigest()
if sha1_ok: self.sha1_data = sha1.hexdigest()
if sha256_ok: self.sha256_data = sha256.hexdigest()
if sha512_ok: self.sha512_data = sha512.hexdigest()
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""{"path": "%s", "stat": %s, "hash": {"md5": "%s", "sha1": "%s", "sha256": "%s", "sha512": "%s"}}\n""",
self.path,
self.st.to_json(),
self.md5_data,
self.sha1_data,
self.sha256_data,
self.sha512_data,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef cypclass SymlinkNode(Node):
string target
int error
void build_node(self, lock cyplist[dev_t] dev_whitelist, lock cyplist[string] ignore_paths):
size = self.st.st_data.st_size + 1
self.target.resize(size)
real_size = readlink(self.path.c_str(), <char*> self.target.data(), size)
self.error = not (0 < real_size < size)
self.target.resize(real_size)
self.format_node()
void format_node(self):
if self.error:
Node.format_node(self)
else:
self.formatted = sprintf("""{"path": "%s", "stat": %s, "target": "%s"}\n""",
self.path,
self.st.to_json(),
self.target,
)
void write_node(self, FILE * stream):
fwrite(self.formatted.data(), 1, self.formatted.size(), stream)
cdef int start(const char *path) nogil:
printf("TEST TEST TEST TEST TEST\n\n") # DEBUG
# TODO replace 4096 by PATH_MAX (yet it will not be perfect)
cdef char resolved_path[4096]
cdef pid_t wait_error = -1 # DEBUG
cdef char* program_name = "fluentbit"
cdef char* arg1 = "-e"
cdef char* arg2 = "/etc/fluentbit_wendelin.so"
cdef char* arg3 = "-c"
cdef char* arg4 = "/etc/flb.conf"
cdef pid_t child_pid = -1 # DEBUG
child_pid = fork() # DEBUG
cdef int err
#cdef char ip_address[100]
#cdef FILE *address_path = fopen("/sys/class/net/ens3/address", "r")
if child_pid == 0: # CHILD
err = execlp("/sbin/fluent-bit", program_name, arg1, arg2, arg3, arg4, 0)
fprintf(stderr, "ERROR with execlp() in CHILD: %d\n", err)
else: # PARENT
printf("WELCOME TO PARENT\n\n") # DEBUG
sleep(2) # TODO error handling ; check if a wait can be made to wait for the child to perform execlp() (instead of the sleep)
global scheduler
scheduler = Scheduler()
ignore_paths = cyplist[string]()
ignore_paths.append(b'/opt/slapgrid')
ignore_paths.append(b'/srv/slapgrid')
dev_whitelist_paths = cyplist[string]()
dev_whitelist_paths.append(b'.')
dev_whitelist_paths.append(b'/')
dev_whitelist_paths.append(b'/boot/efi')
dev_whitelist_paths.append(b'/root')
dev_whitelist = cyplist[dev_t]()
for p in dev_whitelist_paths:
p_stat = Stat(p)
if p_stat is not NULL:
p_dev = p_stat.st_data.st_dev
dev_whitelist.append(p_dev)
realpath(path, resolved_path)
fprintf(stderr, resolved_path) # DEBUG
fprintf(stderr, "\n") # DEBUG
fflush(stderr) # DEBUG
node = make_node(resolved_path, resolved_path)
if node is NULL:
return -1
active_node = activate(consume node)
active_node.build_node(NULL, consume dev_whitelist, consume ignore_paths)
scheduler.finish()
node = consume active_node
#""" # DEBUG
result = fopen('/var/log/metadata_collect.log', 'w')
if result is NULL:
fprintf(stderr, 'Error creating the log file.\n') # DEBUG
fflush(stderr)
return -1
fprintf(stderr, 'Log opened successfully.\n') # DEBUG
fprintf(stderr, "WRITE_NOTE STAGE\n\n") # DEBUG
#fscanf(address_path, "%s", ip_address)
#fclose(address_path)
#fprintf(result, '{"mac_address": "%s"}\n', ip_address)
node.write_node(result)
fprintf(result, '{}\n')
fprintf(result, 'fluentbit_end\n')
fclose(result)
#""" # DEBUG
del scheduler
fprintf(stderr, "WAITING for fluent-bit to end\n\n")
wait_error = wait(NULL) # TODO improve this call (error handling, etc.)
#wait_error = waitpid(child_pid, NULL, 1) # TODO improve this call (error handling, etc.)
fprintf(stderr, "WAITING ENDS\n\n")
fflush(stderr) # DEBUG
return 0
cdef public int main() nogil:
return start(<char*>'/')
#def python_main():
# start(<char*>'.')
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7