Skip to content

M
metadata-collect-agent
Commit 155020e5 authored by Xiaowu Zhang's avatar Xiaowu Zhang
Browse files
Options

draft working version with fluentbit 

1. add tests, but how?
2. use command to add key in bios
3. set bios password
4. how to check network if have no dhcp
...
parent a3033fb8
Hide whitespace changes
Inline Side-by-side
Showing with 184 additions and 399 deletions
debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent 0 → 120000
View file @ 155020e5
/root/metadata-collect-agent/target/x86_64-unknown-linux-musl/release/metadata-collect-agent
\ No newline at end of file
dracut.module/90metadata-collect/flb.conf 0 → 100644
View file @ 155020e5
[SERVICE]
flush 5
[INPUT]
name tail
path /var/log/metadata_collect.log
[output]
name fluentbit_wendelin
match *
streamtool_uri https://softinst143932.host.vifib.net/erp5/portal_ingestion_policies/default
user zope
password pskmtwfn
buffer_type memory
flush_interval 60s
disable_retry_limit true
dracut.module/90metadata-collect/metadata-collect-agent deleted 120000 → 0
View file @ a3033fb8
/home/lle-bout/Projects/metadata-collect-agent/target/x86_64-unknown-linux-musl/release/metadata-collect-agent
\ No newline at end of file
dracut.module/90metadata-collect/module-setup.sh
View file @ 155020e5
......@@ -12,5 +12,8 @@ depends() {
install() {
inst_multiple head ip grep
inst "$moddir"/metadata-collect-agent "/sbin/metadata-collect-agent"
inst "$moddir"/fluent-bit "/sbin/fluent-bit"
inst "$moddir"/flb.conf "/etc/flb.conf"
inst "$moddir"/fluentbit_wendelin.so "/etc/fluentbit_wendelin.so"
inst_hook pre-pivot 10 "$moddir"/collect.sh
}
dracut.module/Makefile
View file @ 155020e5
dracut_module: 90metadata-collect/metadata-collect-agent 90metadata-collect/collect.sh
dracut_module: 90metadata-collect/fluent-bit 90metadata-collect/fluentbit_wendelin.so 90metadata-collect/metadata-collect-agent 90metadata-collect/collect.sh
include collect-sh-template.mk
90metadata-collect/collect.sh:
@ if [ "${ERP5_USER}" = "" ]; then \
echo "Environment variable ERP5_USER not set"; \
exit 1; \
fi
@ if [ "${ERP5_PASS}" = "" ]; then \
echo "Environment variable ERP5_PASS not set"; \
exit 1; \
fi
@ if [ "${ERP5_BASE_URL}" = "" ]; then \
echo "Environment variable ERP5_BASE_URL not set"; \
exit 1; \
fi
echo "$${collect_sh}" >> 90metadata-collect/collect.sh
90metadata-collect/metadata-collect-agent:
cd ../ && ./rust-build-static.bash
.PHONY: clean
90metadata-collect/fluent-bit:
cd ../ && ./fluent-bit-install.sh && cd dracut.module
90metadata-collect/fluentbit_wendelin.so:
cd ../ && ./fluentbit_wendelin_install.sh && cd dracut.module
.PHONY: clean
clean:
rm -fv 90metadata-collect/collect.sh 90metadata-collect/metadata-collect-agent
......
dracut.module/collect-sh-template.mk
View file @ 155020e5
......@@ -3,16 +3,11 @@ define collect_sh :=
. /lib/dracut-lib.sh
DEFAULT_IF="$$(ip route show default | grep -Po "(?<=dev )\w+" | head -n 1)"
DEFAULT_IF_MAC="$$(ip link show "$$DEFAULT_IF" | grep -Po '(?<= )\w{2}:\w{2}:\w{2}:\w{2}:\w{2}:\w{2}(?= )' | head -n 1)"
ERP5_BASE_URL="$(ERP5_BASE_URL)"
ERP5_USER="$(ERP5_USER)"
ERP5_PASS="$(ERP5_PASS)"
/sbin/metadata-collect-agent "$$NEWROOT"
REFERENCE="COMP-MAC-$$(echo "$$DEFAULT_IF_MAC" | sed s/:/-/g).Metadata.Snapshot"
sleep 120
/sbin/metadata-collect-agent "$$NEWROOT" "$$ERP5_USER" "$$ERP5_PASS" "$$REFERENCE" "$$ERP5_BASE_URL"
endef
export collect_sh
\ No newline at end of file
export collect_sh
dracut.module/dracut.conf 0 → 100644
View file @ 155020e5
hostonly=no
hostonly_cmdline=no
kernel_cmdline="root=LABEL=ROOT ip=dhcp rd.neednet=1"
reproducible=yes
compress=xz
uefi=yes
uefi_stub=/usr/lib/systemd/boot/efi/linuxx64.efi.stub
uefi_secureboot_cert=/etc/uefi-key/db.crt
uefi_secureboot_key=/etc/uefi-key/db.key
add_dracutmodules="metadata-collect"
fluent-bit-install.sh 0 → 100755
View file @ 155020e5
apt-get install cmake flex bison build-essential
rm -rf fluent-bit
git clone https://github.com/fluent/fluent-bit.git
cd fluent-bit
git checkout -b v1.7.4 tags/v1.7.4
cd build
cmake -DFLB_DEBUG=On -DFLB_PROXY_GO=On ..
make
make install
BINARY="$(pwd)/bin/fluent-bit"
strip --strip-all "$BINARY"
objdump -T "$BINARY" || true
cd ../../
ln -sf "$BINARY" dracut.module/90metadata-collect/fluent-bit || true
fluentbit_wendelin_install.sh 0 → 100755
View file @ 155020e5
#!/bin/bash
if [ ! -d "fluentbit-plugin-wendelin" ]; then
git clone https://lab.nexedi.com/xiaowu.zhang/fluentbit-plugin-wendelin
fi
cd fluentbit-plugin-wendelin
make install
cp fluentbit_wendelin.so ../dracut.module/90metadata-collect/fluentbit_wendelin.so
install.sh 0 → 100755
View file @ 155020e5
#!/bin/bash
set -eux
apt-get install make autopoint autoconf libtool libattr1.dev musl-tools sbsigntool
wget http://ftp.us.debian.org/debian/pool/main/d/dracut/dracut-core_051-1_amd64.deb
wget http://ftp.us.debian.org/debian/pool/main/d/dracut/dracut_051-1_all.deb
wget http://ftp.us.debian.org/debian/pool/main/d/dracut/dracut-network_051-1_all.deb
apt install ./dracut-core_051-1_amd64.deb ./dracut_051-1_all.deb ./dracut-network_051-1_all.deb
rm dracut-core_051-1_amd64.deb dracut_051-1_all.deb dracut-network_051-1_all.deb
if [ ! -e "$HOME/.cargo/bin" ]; then
curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh
fi
export PATH=$PATH:$HOME/.cargo/bin
rustup target add x86_64-unknown-linux-musl
if [ ! -e /etc/uefi-key ]; then
mkdir /etc/uefi-key
fi
if [ ! -e /etc/uefi-key/db.crt ]; then
openssl req -newkey rsa:2048 -nodes -keyout /etc/uefi-key/db.key -new -x509 -sha256 -days 36500 -subj "/CN=TEST" -out /etc/uefi-key/db.crt
openssl x509 -outform DER -in /etc/uefi-key/db.crt -out /etc/uefi-key/db.cer
fi
cd dracut.module
make install
cd ..
rm -rf dracut_tmp
mkdir dracut_tmp
disk_info=$(/sbin/fdisk -l | grep '^/dev' | cut -d' ' -f1)
disk_info_list=(${disk_info//' '/})
/sbin/e2label ${disk_info_list[1]} ROOT
rm -rf /boot/efi/EFI/Linux/*
dracut --force -c ./dracut.module/dracut.conf --confdir dracut_tmp
cp -r /boot/efi/EFI /EFI
#maybe not necessairy
cp /etc/uefi-key/db.cer /EFI/db.cer
cp /etc/uefi-key/db.cer /boot/efi/db.cer
uefi=$(ls /EFI/Linux)
efibootmgr --quiet --create --disk ${disk_info_list[0]} --label 'debian UEFI' --loader /EFI/Linux/$uefi
main.py deleted 100755 → 0
View file @ a3033fb8
#!/usr/bin/env python3
# Developed with Python 3.8.5
import argparse
import sys
import os
import stat
import traceback
import hashlib
import io
import multiprocessing
import urllib
import time
import urllib3
from msgpack import dumps
import psutil
import posix1e # pylibacl
import certifi
def compute_hashes(entry_path):
with open(entry_path, mode="rb") as f:
md5 = hashlib.md5()
sha1 = hashlib.sha1()
sha256 = hashlib.sha256()
sha512 = hashlib.sha512()
while True:
data = f.read(io.DEFAULT_BUFFER_SIZE)
md5.update(data)
sha1.update(data)
sha256.update(data)
sha512.update(data)
if len(data) < io.DEFAULT_BUFFER_SIZE:
break
return {"md5": md5.hexdigest(),
"sha1": sha1.hexdigest(),
"sha256": sha256.hexdigest(),
"sha512": sha512.hexdigest()}
def stat_result_to_dict(stat_result):
return {
"st_mode": stat_result.st_mode if hasattr(stat_result, "st_mode") else None,
"st_ino": stat_result.st_ino if hasattr(stat_result, "st_ino") else None,
"st_dev": stat_result.st_dev if hasattr(stat_result, "st_dev") else None,
"st_nlink": stat_result.st_nlink if hasattr(stat_result, "st_nlink") else None,
"st_uid": stat_result.st_uid if hasattr(stat_result, "st_uid") else None,
"st_gid": stat_result.st_gid if hasattr(stat_result, "st_gid") else None,
"st_size": stat_result.st_size if hasattr(stat_result, "st_size") else None,
"st_atime": stat_result.st_atime if hasattr(stat_result, "st_atime") else None,
"st_mtime": stat_result.st_mtime if hasattr(stat_result, "st_mtime") else None,
"st_ctime": stat_result.st_ctime if hasattr(stat_result, "st_ctime") else None,
"st_blocks": stat_result.st_blocks if hasattr(stat_result, "st_blocks") else None,
"st_blksize": stat_result.st_blksize if hasattr(stat_result, "st_blksize") else None,
"st_rdev": stat_result.st_rdev if hasattr(stat_result, "st_rdev") else None,
"st_flags": stat_result.st_flags if hasattr(stat_result, "st_flags") else None,
"st_gen": stat_result.st_gen if hasattr(stat_result, "st_gen") else None,
"st_birthtime": stat_result.st_birthtime if hasattr(stat_result, "st_birthtime") else None,
}
def construct_fs_tree(mp_pool=None, mp_tasks=[], cur_dict=None, path="/", dev_whitelist=None, ignored_dirs=[]):
is_first_call = False
if mp_pool == None:
is_first_call = True
mp_pool = multiprocessing.Pool()
if cur_dict == None:
cur_dict = {"stat": stat_result_to_dict(os.stat(path, follow_symlinks=False)),
"childs": dict()}
if dev_whitelist != None:
path_stat = cur_dict["stat"]
if "st_dev" in path_stat:
if not path_stat["st_dev"] in dev_whitelist:
return cur_dict
for dir in ignored_dirs:
if path.startswith(dir):
cur_dict["ignored"] = True
return cur_dict
try:
with os.scandir(path) as it:
for entry in it:
try:
entry_path = os.fsdecode(entry.path)
entry_name = os.fsdecode(entry.name)
try:
entry_stat = os.stat(entry_path, follow_symlinks=False)
except Exception:
traceback.print_exc()
entry_stat = None
cur_dict["childs"][entry_name] = {"stat": stat_result_to_dict(entry_stat),
"childs": dict()}
try:
cur_dict["childs"][entry_name]["xattrs"] = dict()
for k in os.listxattr(entry_path, follow_symlinks=False):
cur_dict["childs"][entry_name]["xattrs"][k] = os.getxattr(
entry_path, k, follow_symlinks=False)
except Exception:
traceback.print_exc()
try:
cur_dict["childs"][entry_name]["posix_acls"] = posix1e.ACL(
file=entry_path).to_any_text(options=posix1e.TEXT_ALL_EFFECTIVE)
except Exception:
traceback.print_exc()
if stat.S_ISDIR(entry_stat.st_mode):
construct_fs_tree(mp_pool=mp_pool, mp_tasks=mp_tasks, cur_dict=cur_dict["childs"][entry_name],
path=entry_path, dev_whitelist=dev_whitelist)
elif stat.S_ISREG(entry_stat.st_mode):
mp_tasks.append({"result": mp_pool.apply_async(compute_hashes, [entry_path]),
"merge_into": cur_dict["childs"][entry_name]})
elif stat.S_ISLNK(entry_stat.st_mode):
cur_dict["childs"][entry_name]["symlink_target"] = os.readlink(
entry_path)
except Exception:
traceback.print_exc()
except Exception:
traceback.print_exc()
if is_first_call == True:
mp_pool.close()
for task in mp_tasks:
try:
result = task["result"].get()
for k in iter(result):
task["merge_into"][k] = result[k]
except Exception:
traceback.print_exc()
mp_pool.join()
return cur_dict
def upload_to_erp5(fileobj, size, base_url, username, password, reference):
pool = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',
ca_certs=certifi.where())
url = '{0}/portal_contributions/newContent'.format(base_url)
headers = urllib3.make_headers(
keep_alive=True, basic_auth='{0}:{1}'.format(username, password))
resp = pool.request('GET', url, fields={'portal_type': 'File',
'filename': '{0}.metadata'.format(reference),
'container_path': 'document_module',
'data': ''},
headers=headers,
redirect=False)
# workaround for ERP5 disappearing documents race condition bug
time.sleep(3)
upload_path = resp.headers['X-Location']
data = fileobj.read(io.DEFAULT_BUFFER_SIZE)
offset = 0
upload_headers = headers
while data:
upload_headers['Content-Range'] = 'bytes {0}-{1}/{2}'.format(
offset, offset+len(data), size)
pool.urlopen('PUT', upload_path, headers=upload_headers, body=data)
offset += len(data)
data = fileobj.read(io.DEFAULT_BUFFER_SIZE)
parsed = urllib.parse.urlparse(upload_path)
resp = pool.request('POST', upload_path, fields={
'form_id': 'File_view',
'object_path': parsed.path,
'field_my_reference': reference,
'Base_edit:method': '',
},
redirect=False,
headers=headers)
# workaround for ERP5 disappearing documents race condition bug
time.sleep(3)
resp = pool.request('POST', upload_path, fields={
'dialog_id': 'Base_viewWorkflowActionDialog',
'dialog_method': 'Workflow_statusModify',
'form_id': 'File_view',
'object_path': parsed.path,
'field_your_comment': '',
'field_your_workflow_action': 'share_action',
'Base_callDialogMethod:method': '',
},
redirect=False,
headers=headers)
# workaround for ERP5 disappearing documents race condition bug
time.sleep(3)
def main():
parser = argparse.ArgumentParser(
description="Collect and report metadata about a system")
parser.add_argument("start_directory", type=str, default="/")
parser.add_argument("--ignored-dirs", type=str, nargs="+", default=[])
parser.add_argument("--erp5-user", type=str)
parser.add_argument("--erp5-pass", type=str)
parser.add_argument("--erp5-file-reference", type=str)
parser.add_argument("--erp5-base-url", type=str)
args = parser.parse_args()
parts = psutil.disk_partitions(all=False)
dev_whitelist = list()
for part in parts:
dev_whitelist.append(
os.stat(part.mountpoint, follow_symlinks=False).st_dev)
tree = construct_fs_tree(path=args.start_directory,
dev_whitelist=dev_whitelist, ignored_dirs=args.ignored_dirs)
final = {'disk_partitions': parts, 'fs_tree': tree}
packed = dumps(final)
upload_to_erp5(io.BytesIO(packed), len(packed), args.erp5_base_url,
args.erp5_user, args.erp5_pass, args.erp5_file_reference)
if __name__ == "__main__":
main()
mkkeys.sh 0 → 100755
View file @ 155020e5
#!/bin/bash
# Copyright (c) 2015 by Roderick W. Smith
# Licensed under the terms of the GPL v3
echo -n "Enter a Common Name to embed in the keys: "
read NAME
openssl req -new -x509 -newkey rsa:2048 -subj "/CN=$NAME PK/" -keyout PK.key \
-out PK.crt -days 3650 -nodes -sha256
openssl req -new -x509 -newkey rsa:2048 -subj "/CN=$NAME KEK/" -keyout KEK.key \
-out KEK.crt -days 3650 -nodes -sha256
openssl req -new -x509 -newkey rsa:2048 -subj "/CN=$NAME DB/" -keyout DB.key \
-out DB.crt -days 3650 -nodes -sha256
openssl x509 -in PK.crt -out PK.cer -outform DER
openssl x509 -in KEK.crt -out KEK.cer -outform DER
openssl x509 -in DB.crt -out DB.cer -outform DER
GUID=`python3 -c 'import uuid; print(str(uuid.uuid1()))'`
echo $GUID > myGUID.txt
cert-to-efi-sig-list -g $GUID PK.crt PK.esl
cert-to-efi-sig-list -g $GUID KEK.crt KEK.esl
cert-to-efi-sig-list -g $GUID DB.crt DB.esl
rm -f noPK.esl
touch noPK.esl
sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k PK.key -c PK.crt PK PK.esl PK.auth
sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k PK.key -c PK.crt PK noPK.esl noPK.auth
sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k PK.key -c PK.crt KEK KEK.esl KEK.auth
sign-efi-sig-list -t "$(date --date='1 second' +'%Y-%m-%d %H:%M:%S')" \
-k KEK.key -c KEK.crt db DB.esl DB.auth
chmod 0600 *.key
echo ""
echo ""
echo "For use with KeyTool, copy the *.auth and *.esl files to a FAT USB"
echo "flash drive or to your EFI System Partition (ESP)."
echo "For use with most UEFIs' built-in key managers, copy the *.cer files;"
echo "but some UEFIs require the *.auth files."
echo ""
rust-build-static.bash
View file @ 155020e5
......@@ -31,5 +31,5 @@ BINARY="$(pwd)/target/$HOST_TARGET/release/$(basename "$(pwd)")"
strip --strip-all "$BINARY"
objdump -T "$BINARY" || true
ln -s "$BINARY" dracut.module/90metadata-collect/metadata-collect-agent || true
ln -s "$BINARY" debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent || true
\ No newline at end of file
ln -sf "$BINARY" dracut.module/90metadata-collect/metadata-collect-agent || true
ln -sf "$BINARY" debian.package.unsafe/unsafe-boot-metadata-collect-agent/sbin/metadata-collect-agent || true
setup.py deleted 100644 → 0
View file @ a3033fb8
#!/usr/bin/env python3
from setuptools import setup, find_packages
setup(
name="metadata-collect-agent",
version="0.1",
packages=find_packages(),
scripts=["main.py"],
install_requires=["pylibacl>=0.5.4",
"msgpack>=1.0.0",
"urllib3>=1.25.10"
"psutil>=5.7.2"],
author="Nexedi SA",
author_email="leo.le.bouter@nexedi.com"
)
src/main.rs
View file @ 155020e5
use anyhow::Result;
use blocking::multipart;
use clap::{App, Arg};
use multipart::Part;
use rayon::prelude::*;
use reqwest::{
blocking::{self, Client},
header, redirect,
};
use serde::Serialize;
use std::collections::HashMap;
use std::{
ffi::OsString,
fs::DirEntry,
io::{Cursor, Read},
path::PathBuf,
sync::{Arc, Mutex},
thread,
time::Duration,
};
use std::process::Command;
use std::io::prelude::*;
use std::fs::OpenOptions;
#[derive(Debug, Serialize)]
struct FileStat {
st_dev: u64,
......@@ -212,84 +208,6 @@ fn construct_fs_tree(
Ok(cur_tree)
}
fn upload_to_erp5<R: Read + Send + 'static>(
reader: R,
size: u64,
base_url: &str,
username: &str,
password: &str,
reference: &str,
) -> Result<()> {
let mut headers = header::HeaderMap::new();
headers.insert(
header::AUTHORIZATION,
header::HeaderValue::from_str(&format!(
"Basic {}",
&base64::encode(&format!("{}:{}", username, password))
))?,
);
let client = Client::builder()
.redirect(redirect::Policy::none())
.default_headers(headers)
.build()?;
let resp = client
.get(&format!("{}/portal_contributions/newContent", base_url))
.query(&[
("portal_type", "Computer Metadata Snapshot"),
("filename", &format!("{}.metadata", &reference)),
("container_path", "computer_metadata_snapshot_module"),
("data", ""),
])
.send()?;
// workaround for ERP5 disappearing documents race condition bug
thread::sleep(Duration::from_secs(3));
let doc_url: reqwest::Url = resp
.headers()
.get("X-Location")
.ok_or_else(|| anyhow::format_err!("X-Location must exist"))?
.to_str()?
.parse()?;
let _resp = client
.post(doc_url.to_owned())
.multipart(
multipart::Form::new()
.text("form_id", "File_view")
.text("object_path", doc_url.path().to_owned())
.text("field_my_reference", reference.to_owned())
.part(
"field_my_file",
Part::reader_with_length(reader, size)
.file_name(format!("{}.metadata", &reference)),
)
.text("Base_edit:method", ""),
)
.send()?;
// workaround for ERP5 disappearing documents race condition bug
thread::sleep(Duration::from_secs(3));
let mut params = HashMap::new();
params.insert("dialog_id", "Base_viewWorkflowActionDialog");
params.insert("dialog_method", "Workflow_statusModify");
params.insert("form_id", "File_view");
params.insert("object_path", doc_url.path());
params.insert("field_your_comment", "");
params.insert("field_your_workflow_action", "share_action");
params.insert("Base_callDialogMethod:method", "");
let _resp = client.post(doc_url.to_owned()).form(&params).send()?;
// workaround for ERP5 disappearing documents race condition bug
thread::sleep(Duration::from_secs(3));
Ok(())
}
fn main() -> Result<()> {
let m = App::new("metadata-collect-agent")
......@@ -298,22 +216,6 @@ fn main() -> Result<()> {
.required(true)
.takes_value(true)
.multiple(false),
Arg::with_name("erp5-user")
.takes_value(true)
.multiple(false)
.required(true),
Arg::with_name("erp5-pass")
.takes_value(true)
.multiple(false)
.required(true),
Arg::with_name("erp5-file-reference")
.takes_value(true)
.multiple(false)
.required(true),
Arg::with_name("erp5-base-url")
.takes_value(true)
.multiple(false)
.required(true),
Arg::with_name("ignored-dirs")
.takes_value(true)
.multiple(true)
......@@ -326,6 +228,8 @@ fn main() -> Result<()> {
.unwrap_or(clap::Values::default())
.map(PathBuf::from)
.collect();
let mut file = OpenOptions::new().write(true).append(true).create(true).open("/var/log/metadata_collect.log").unwrap();
let mut child = Command::new("/sbin/fluent-bit").args(&["-e","/etc/fluentbit_wendelin.so","-c","/etc/flb.conf"]).spawn().unwrap();
let disk_partitions = psutil::disk::partitions_physical()?;
......@@ -343,18 +247,13 @@ fn main() -> Result<()> {
&ignored_dirs,
)?,
};
let packed = serde_json::to_vec(&snapshot)?;
let packed_size = packed.len() as u64;
upload_to_erp5(
Cursor::new(packed),
packed_size,
m.value_of("erp5-base-url").unwrap(),
m.value_of("erp5-user").unwrap(),
m.value_of("erp5-pass").unwrap(),
m.value_of("erp5-file-reference").unwrap(),
)?;
let packed = serde_json::to_string(&snapshot)?;
for s in packed.split(":{"){
file.write_all((s.to_owned()+"\n").as_bytes()).expect("Unable to write to log");
}
file.write_all("fluentbit_end\n".as_bytes()).expect("Unable to write to log");
println!("finished to write file");
let _result = child.wait().unwrap();
Ok(())
}
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7