Commit 781bac92 authored by Ophélie Gagnard's avatar Ophélie Gagnard

Remove useless files for minimal installation.

parent 9514ca47
#!/bin/bash
check() {
require_binaries grep head ip || return 1
return 255
}
depends() {
echo network
}
install() {
inst_multiple head ip grep
inst "$moddir"/metadata-collect-agent "/sbin/metadata-collect-agent"
inst "$moddir"/fluent-bit "/sbin/fluent-bit"
inst "$moddir"/flb.conf "/etc/flb.conf"
inst "$moddir"/fluentbit_wendelin.so "/etc/fluentbit_wendelin.so"
inst_hook pre-pivot 10 "$moddir"/collect.sh
}
define collect_sh :=
#!/bin/bash
. /lib/dracut-lib.sh
>&2 echo "$$NEWROOT"
/sbin/metadata-collect-agent "$$NEWROOT"
endef
export collect_sh
hostonly=no
hostonly_cmdline=no
kernel_cmdline="root=LABEL=ROOT ip=dhcp rd.neednet=1"
reproducible=yes
compress=xz
uefi=yes
uefi_stub=/usr/lib/systemd/boot/efi/linuxx64.efi.stub
add_dracutmodules="metadata-collect"
hostonly=no
hostonly_cmdline=no
kernel_cmdline="root=LABEL=ROOT ip=dhcp rd.neednet=1"
reproducible=yes
compress=xz
uefi=yes
uefi_stub=/usr/lib/systemd/boot/efi/linuxx64.efi.stub
uefi_secureboot_cert=/etc/uefi-keys/db.crt
uefi_secureboot_key=/etc/uefi-keys/db.key
add_dracutmodules="metadata-collect"
.PHONY: all
all: 90metadata-collect/fluent-bit 90metadata-collect/fluentbit_wendelin.so 90metadata-collect/metadata-collect-agent 90metadata-collect/collect.sh
include collect-sh-template.mk
90metadata-collect/collect.sh:
echo "$${collect_sh}" >> 90metadata-collect/collect.sh
90metadata-collect/metadata-collect-agent:
cd ../scan-filesystem/cython/ && make nopython && cd ../../dracut.module
90metadata-collect/fluent-bit:
cd ../ && ./fluent-bit-install.sh && cd dracut.module
90metadata-collect/fluentbit_wendelin.so:
cd ../ && ./fluentbit_wendelin_install.sh && cd dracut.module
.PHONY: clean
clean:
rm -fv 90metadata-collect/collect.sh 90metadata-collect/metadata-collect-agent
.PHONY: install
install: all
cp -vLr "90metadata-collect" /usr/lib/dracut/modules.d
.PHONY: uninstall
uninstall:
rm -rfv /usr/lib/dracut/modules.d/90metadata-collect
if [ ! -d fluent-bit ]; then
git clone https://github.com/fluent/fluent-bit.git
fi
cd fluent-bit
git checkout -b v1.7.4 tags/v1.7.4
cd build
cmake -DFLB_DEBUG=On -DFLB_PROXY_GO=On ..
make
make install
BINARY="$(pwd)/bin/fluent-bit"
strip --strip-all "$BINARY"
objdump -T "$BINARY" || true
cd ../../
#ln -sf "$BINARY" dracut.module/90metadata-collect/fluent-bit || true
#!/bin/bash
set -eux
# apt install git
# installing miscellaneous useful packages
apt -y install make autopoint autoconf libtool libattr1-dev musl-tools mmv sbsigntool
# installing more necessary packages to activate Secure Boot with our own keys
apt -y install efitools
# installing packages needed to build fluentbit
apt -y cmake flex bison build-essential
# installing (download and compile) a specific version of python to make sure the whole installation process work
: '
if [ ! -d "/opt/python-3.7.12/include/python3.7m" ]; then
wget https://www.python.org/ftp/python/3.7.12/Python-3.7.12.tgz
tar zxf Python-3.7.12.tgz
cd Python-3.7.12/
./configure --prefix=/opt/python-3.7.12
make
make install
cd ..
fi
rm -f Python-3.7.12.tgz
rm -rf Python-3.7.12/
#'
: '
## Install dracut
if ! which dracut ; then
wget http://ftp.us.debian.org/debian/pool/main/d/dracut/dracut-core_051-1_amd64.deb
wget http://ftp.us.debian.org/debian/pool/main/d/dracut/dracut_051-1_all.deb
wget http://ftp.us.debian.org/debian/pool/main/d/dracut/dracut-network_051-1_all.deb
apt -y install ./dracut-core_051-1_amd64.deb ./dracut_051-1_all.deb ./dracut-network_051-1_all.deb
rm dracut*.deb
fi
#'
## Install others
apt -y install libssl-dev
# needed to build fmt library
apt -y install cmake
## Install Go
# needed for dracut.module/ make install
#GOVERSION=1.16.5
#if [ ! -e "/usr/local/go/VERSION" ] || [ go${GOVERSION} != $(cat /usr/local/go/VERSION) ]; then
# wget https://golang.org/dl/go${GOVERSION}.linux-amd64.tar.gz
# rm -rf /usr/local/go
# tar -C /usr/local -xzf go${GOVERSION}.linux-amd64.tar.gz
# rm go*.tar.gz*
# # TODO: write in /etc/profile.d/ instead
# echo -e "\n" >> /etc/profile
# echo 'export PATH=$PATH:/usr/local/go/bin' >> /etc/profile
# source /etc/profile
#fi
apt -y install golang
## Install Cython+
# needed for scan-filesystem/ make nopython
# WARNING: it does not differentiate Cython+ from Cython
: '
if ! which cython3 ; then
mkdir -p build_cythonplus && cd build_cythonplus
wget https://lab.nexedi.com/nexedi/cython/raw/debian/build_cythonplus
chmod 744 build_cythonplus
./build_cythonplus
apt -y install ./cython3_*
cd ..
fi
'
apt -y install libfmt-dev
: '
## Install fmt library
# needed for scan-filesystem/ make nopython
if [ ! -d "fmt" ]; then
git clone https://github.com/fmtlib/fmt.git
cd fmt && mkdir -p build_fmt && cd build_fmt
cmake -DCMAKE_POSITION_INDEPENDENT_CODE=TRUE ..
make
make install
cd ../..
fi
'
: '
cd dracut.module
make install
cd ..
'
apt -y autoremove
echo -e '\nInstallation finished.\nType ^C (ctrl+C) to exit.\nType Enter to proceed to building the UEFI application.\n'
read
## Build UEFI application
if [ ! -e uefi-key ]; then
mkdir uefi-key
cd uefi-key
../mkkeys.sh
cd ..
fi
if [ ! -e /etc/uefi-key ]; then
mkdir /etc/uefi-key
cp uefi-key/DB.cer /etc/uefi-key/db.cer
cp uefi-key/DB.crt /etc/uefi-key/db.crt
cp uefi-key/DB.key /etc/uefi-key/db.key
fi
# TODO: write a separate script: one to install stuff, the other to build the UEFI application
: '
# creates the keys needed to sign the UEFI application if they do not exist
# when creating and copying (or simply copying) the keys in /etc/uefi-key/ by hand,
# the following lines wont be executed
if [ ! -e /etc/uefi-key/db.crt ]; then
openssl req -newkey rsa:2048 -nodes -keyout /etc/uefi-key/db.key -new -x509 -sha256 -days 36500 -subj "/CN=TEST" -out /etc/uefi-key/db.crt
openssl x509 -outform DER -in /etc/uefi-key/db.crt -out /etc/uefi-key/db.cer
fi
#'
#source secure-boot-automation.sh
rm -rf dracut_tmp
mkdir dracut_tmp
disk_info=$(/sbin/fdisk -l | grep '^/dev' | cut -d' ' -f1)
disk_info_list=(${disk_info//' '/})
/sbin/e2label ${disk_info_list[4]} ROOT
#rm -rf /boot/efi/EFI/Linux/*
mmv '/boot/efi/EFI/Linux/*.*' '/boot/efi/EFI/Linux/#1_1.#2'
dracut --force -c ./dracut.module/dracut.conf --confdir dracut_tmp
cp -r /boot/efi/EFI /EFI
# the following two lines are useful if one wants to enroll the keys from the BIOS GUI (and not from command-line as root) (??)
cp /etc/uefi-keys/db.cer /EFI/db.cer
cp /etc/uefi-keys/db.cer /boot/efi/db.cer
uefi=$(echo $(ls -t /EFI/Linux/ | cut -d' ' -f1) | cut -d' ' -f1)
efibootmgr --quiet --create --disk ${disk_info_list[3]} --label 'debian UEFI test_root' --loader /EFI/Linux/$uefi
ifeq (,)
INCLUDE_PYTHON = $(shell pkg-config --cflags python3 | cut -f 1 -d " ")
else
INCLUDE_PYTHON = -I/srv/slapgrid/slappart6/srv/runner/shared/python3/2e435adf7e2cb7d97668da52532ac7f3/include/python3.7m
OPENSSL_PATH = /srv/slapgrid/slappart6/srv/runner/shared/openssl/24bd61db512fe6e4e0d214ae77943d75
INCLUDE_OPENSSL = -I$(OPENSSL_PATH)/include
LIBRARY_OPENSSL = -L$(OPENSSL_PATH)/lib
RUNPATH_OPENSSL = -Wl,-rpath=$(OPENSSL_PATH)/lib
FMTLIB_PATH = /srv/slapgrid/slappart6/srv/runner/shared/fmtlib/d524cc3d1a798a140778558556ec6d0c
INCLUDE_FMTLIB = -I$(FMTLIB_PATH)/include
LIBRARY_FMTLIB = -L$(FMTLIB_PATH)/lib
RUNPATH_FMTLIB = -Wl,-rpath=$(FMTLIB_PATH)/lib
INCLUDE_DIRS = $(INCLUDE_PYTHON) $(INCLUDE_OPENSSL) $(INCLUDE_FMTLIB)
LIBPATHS = $(LIBRARY_OPENSSL) $(LIBRARY_FMTLIB)
RUNPATHS = $(RUNPATH_OPENSSL) $(RUNPATH_FMTLIB)
LDFLAGS = $(LIBPATHS) $(RUNPATHS)
endif
#EXE = main
#CXX = g++
#CPPFLAGS = -O2 -g -Wno-unused-result -Wsign-compare -pthread $(INCLUDE_DIRS)
#LDFLAGS += -Wl,--unresolved-symbols=ignore-all
LDLIBS = -lcrypto -lfmt
EXT_SUFFIX := $(shell python3 -c "import sysconfig; print(sysconfig.get_config_var('EXT_SUFFIX'))")
EXT = $(EXE)$(EXT_SUFFIX)
# Build without Python runtime
all: main.cpp#$(EXE)
mkdir -p logs
# Error expected here, writing a C program using the error messages
-g++ -O2 -g -Wno-unused-result -Wsign-compare -pthread $(INCLUDE_PYTHON) main.cpp -lcrypto -lfmt -o main 2> logs/link_errors
./parse_link_errors.py < logs/link_errors
make fake_python.o
g++ -O2 -g -Wno-unused-result -Wsign-compare -pthread $(INCLUDE_PYTHON) main.cpp fake_python.o -lcrypto -lfmt -o metadata-collect-agent
#-ln -f ../../metadata-collect-agent metadata-collect-agent
-ln -f metadata-collect-agent ../../dracut.module/90metadata-collect/metadata-collect-agent
#-ln -f metadata-collect-agent ../../debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent
%.cpp: %.pyx
@echo "[Cython Compiling $^ -> $@]"
python3 -c "from Cython.Compiler.Main import main; main(command_line=1)" $^ --cplus -3
@rm -f $(subst .cpp,.h,$@)
%: %.cpp
@echo "[C++ Compiling $^ -> $@]"
$(LINK.cpp) $^ $(LOADLIBES) $(LDLIBS) -o $@
clean:
-rm -f *.c *.cpp *.html
-rm -f *.h
-rm -f *.so
-rm -f $(EXE)
-rm -f *.o
-rm -f -r build
-rm -f *.json
-rm -f /var/log/metadata_collect.log
-rm -f metadata-collect-agent
-rm -f ../../dracut.module/90metadata-collect/metadata-collect-agent
-rm -f ../../debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent
-rm -rf logs
.PHONY: all clean
.PRECIOUS: %.cpp
#!/bin/bash
# Copyright (c) 2015 by Roderick W. Smith
# Licensed under the terms of the GPL v3
echo -n "Enter a Common Name to embed in the keys: "
read NAME
openssl req -new -x509 -newkey rsa:2048 -subj "/CN=$NAME PK/" -keyout PK.key \
-out PK.crt -days 3650 -nodes -sha256
openssl req -new -x509 -newkey rsa:2048 -subj "/CN=$NAME KEK/" -keyout KEK.key \
-out KEK.crt -days 3650 -nodes -sha256
openssl req -new -x509 -newkey rsa:2048 -subj "/CN=$NAME DB/" -keyout DB.key \
-out DB.crt -days 3650 -nodes -sha256
openssl x509 -in PK.crt -out PK.cer -outform DER
openssl x509 -in KEK.crt -out KEK.cer -outform DER
openssl x509 -in DB.crt -out DB.cer -outform DER
GUID=`python3 -c 'import uuid; print(str(uuid.uuid1()))'`
echo $GUID > myGUID.txt
cert-to-efi-sig-list -g $GUID PK.crt PK.esl
cert-to-efi-sig-list -g $GUID KEK.crt KEK.esl
cert-to-efi-sig-list -g $GUID DB.crt DB.esl
rm -f noPK.esl
touch noPK.esl
sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k PK.key -c PK.crt PK PK.esl PK.auth
sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k PK.key -c PK.crt PK noPK.esl noPK.auth
sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k PK.key -c PK.crt KEK KEK.esl KEK.auth
sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k KEK.key -c KEK.crt db DB.esl DB.auth
chmod 0600 *.key
echo ""
echo ""
echo "For use with KeyTool, copy the *.auth and *.esl files to a FAT USB"
echo "flash drive or to your EFI System Partition (ESP)."
echo "For use with most UEFIs' built-in key managers, copy the *.cer files;"
echo "but some UEFIs require the *.auth files."
echo ""
#!/bin/bash
rm -rf dracut_tmp
mkdir dracut_tmp
disk_info=$(/sbin/fdisk -l | grep '^/dev' | cut -d' ' -f1)
disk_info_list=(${disk_info//' '/})
/sbin/e2label ${disk_info_list[4]} ROOT
#rm -rf /boot/efi/EFI/Linux/*
mmv '/boot/efi/EFI/Linux/*.*' '/boot/efi/EFI/Linux/#1_1.#2'
dracut --force -c ./dracut.module/dracut.conf --confdir dracut_tmp
cp -r /boot/efi/EFI /EFI
# the following two lines are useful if one wants to enroll the keys from the BIOS GUI (and not from command-line as root) (??)
#cp /etc/uefi-keys/db.cer /EFI/db.cer
#cp /etc/uefi-keys/db.cer /boot/efi/db.cer
uefi=$(echo $(ls -t /EFI/Linux/ | cut -d' ' -f1) | cut -d' ' -f1)
efibootmgr --quiet --create --disk ${disk_info_list[3]} --label 'debian UEFI test_root' --loader /EFI/Linux/$uefi
# metadata-collect-agent
In the context of the project [GNU/Linux System files on-boot Tamper Detection System](https://www.erp5.com/group_section/forum/GNU-Linux-System-files-on-boot-Tamper-Detection-System-94DGdYfmx1), we need to create an agent that will be run inside an initramfs to report as much metadata as useful while keeping system boot times acceptable. It must then report that metadata to a remote service for later analysis.
## Current performance properties
- Reads file system metadata from the main thread (stat, xattrs (SELinux, ..), POSIX ACLs)
- Reads files and hashes them in md5, sha1, sha256 and sha512 across multiple processes (as many as core count) with the `multiprocessing` python module
- Maximizes disk I/O utilization successfully, the Python code's performance is not a bottleneck, the disk is (good sign)
Tested on a laptop with:
- 3.2 GB/s read NVMe SSD
- Intel(R) Core(TM) i7-1065G7 CPU (4 cores, 8 threads) @ 1.30 GHz min / 3.90 GHz max
- 2 GHz per thread on average under full multithreaded load due to heat and unoptimal laptop thermals (Dell XPS 13 2020)
- For ~1 million files on EXT4 over LUKS+LVM and ~140GB occupied disk space:
```
real 6m11.532s
user 31m7.676s
sys 3m27.251s
```
6 minutes and 12 seconds of real world time
This will hardly get any better because the disk is the bottleneck, CPU usage is not full but disk I/O utilization is, peaking at 500 MB/s reads for these test conditions. 3.2 GB/s on this SSD is for sequential reads (optimal conditions).
It can and probably will be faster on performant servers with less files, less disk space usage, more CPU cores and similar disk.
## Desired performance properties
- Reduce memory usage
- Avoid storing all the collected data in memory at the same time
- encode and output JSON as the program runs (incompatible with tree-like data structure like now)
- discard data after output so that memory usage can be deterministic
- Beware of stack overflows
- Currently the file system traverse function is recursive, Python does not have tail recursion optimization so it potentially could overflow the stack. But due to file system paths being limited in size (is it always true? is it file system specific?), probably it's unlikely it ever will.
\ No newline at end of file
# To perform the following instructions, some packages are needed.
# They are all installed with apt at the begining of install.sh.
## Create the keys:
# run the script mkkeys.sh (better in a dedicated directory)
mkdir -p generate_keys && cd generate_keys && cp ../mkkeys.sh . && ./mkkeys.sh
## Signing a UEFI application:
# before running the following line (from install.sh)
dracut --force -c ./dracut.module/dracut.conf --confdir dracut_tmp
# put the following from the signing key in /etc/uefi-key/
# db.cer db.crt db.key
## Add keys to the firmware:
# run
efi-readvar
# if PK, KEK or db is not empty then
# reboot and, in the BIOS, disable Secure Boot (SB) and remove every keys
# then if we assume the keys have the name mkkeys.sh gives them:
# Add the Platform Key (the authority key for the machine).
efi-updatevar -f PK.auth PK
# Add the Key Exchange Key (the authority key for the db and dbx certificates and hashes)
efi-updatevar -a -c KEK.crt -k PK.key KEK
# Replace the current whitelist certificate
efi-updatevar -f -c DB.crt -k KEK.key db
# (replace -f with -a to append another whitelist certificate instead of replacing the old one)
### Troubleshooting
# Some general informations:
# 1) As a general matter, with Secure Boot (SB) disabled and no key, every key should addable.
# 2) A Platform Key is needed for Secure Boot (SB) to be enabled.
# 3) To add or modify a key, you need the private key of the upper level (the Platform Key is its own upper level key).
## Common errors:
# error: "Failed to update <var>: Operation not permitted"
# when trying to delete or replace a key, may be resolved by making it mutable:
lsattr /sys/firmware/efi/efivars/{db,dbx,KEK,PK}*
# "----i--------------" before the key means it is set as immutable
# to make it mutable
chattr -i /sys/firmware/efi/efivars/<key>
# (example:)
chattr -i /sys/firmware/efi/efivars/db-d719b2cb-3d3a-4596-a3bc-dad00e67656f
# recommanded: make everything immutable after your operation
# to do so, perform the same command with +i instead of -i (same example:)
chattr +i /sys/firmware/efi/efivars/db-d719b2cb-3d3a-4596-a3bc-dad00e67656f
# error: "Cannot write to db, wrong filesystem permissions"
# may be resolved providing the upper level private key
# (note that it can happen even with root mode, as keys are stored on a separate filesystem)
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment