software/ors-amarisoft: Send xlog with fluentbit

See merge request nexedi/slapos!1415

software/ors-amarisoft/buildout.hash.cfg

@@ -16,7 +16,7 @@
 
 [template]
 filename = instance.cfg
-md5sum = 38bceddc77a5a44d69f0d9614909d1a2
+md5sum = efcb5965bdf766d94b43c6ba2328275d
 
 [amarisoft-stats.jinja2.py]
 _update_hash_filename_ = amarisoft-stats.jinja2.py
@@ -36,11 +36,11 @@ md5sum = b34fe47a73890097fbc6ea6374aeb38d
 
 [template-enb]
 _update_hash_filename_ = instance-enb.jinja2.cfg
-md5sum = 6382f871c0f4e7e965c95de7a959342a
+md5sum = 8b6e778ec21e03ef3f832ee420d7e83a
 
 [template-gnb]
 _update_hash_filename_ = instance-gnb.jinja2.cfg
-md5sum = 4df8edfb9a8bcbdcc9740afb27a88928
+md5sum = a94322f85f438dccba699e0f5ea09e3a
 
 [template-core-network]
 _update_hash_filename_ = instance-core-network.jinja2.cfg

software/ors-amarisoft/instance-enb-input-schema.json.jinja2

@@ -276,6 +276,21 @@
       "description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
       "type": "number",
       "default": 7
+    },
+    "xlog_fluentbit_forward_host": {
+      "title": "Address to Forward Xlog by Fluenbit",
+      "description": "Address of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_port": {
+      "title": "Port to Forward Xlog by Fluentbit",
+      "description": "Optional Port of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_shared_key": {
+      "title": "Shared Key to Forward Xlog by Fluentbit",
+      "description": "Secret Key Shared with Remote Fluentd or Fluentbit Server for Authentication when Forwarding Xlog",
+      "type": "string"
     }
   }
 }

software/ors-amarisoft/instance-enb.jinja2.cfg

@@ -5,6 +5,9 @@ parts =
   enb-config
   enb-service
   xamari-xlog-service
+{% if slapparameter_dict.get('xlog_fluentbit_forward_host') %}
+  xlog-fluentbit-service
+{% endif %}
   amarisoft-stats-service
   amarisoft-rf-info-service
 {% if slapparameter_dict.get('rrh', '') == "Lopcomm ORAN" %}
@@ -176,6 +179,43 @@ wrapper-path = ${directory:service}/${:_buildout_section_name_}
 command-line = ${xamari-xlog-script:output}
 hash-files = ${:command-line}
 
+{% if slapparameter_dict.get('xlog_fluentbit_forward_host') %}
+[xlog-fluentbit-config]
+recipe = slapos.recipe.template
+output = ${directory:etc}/${:_buildout_section_name_}.cfg
+logfile = ${xamari-xlog-script:logfile}
+forward-host = {{ slapparameter_dict.get('xlog_fluentbit_forward_host', '') }}
+forward-port = {{ slapparameter_dict.get('xlog_fluentbit_forward_port', '') }}
+forward-shared-key = {{ slapparameter_dict.get('xlog_fluentbit_forward_shared_key', '') }}
+forward-self-hostname = {{ ors_id['ors-id'] }}
+inline =
+  [SERVICE]
+      flush           5
+  [INPUT]
+      name            tail
+      path            ${:logfile}
+      Read_from_Head  True
+  [OUTPUT]
+      name            forward
+      match           *
+      Host            ${:forward-host}
+{%- if slapparameter_dict.get('xlog_fluentbit_forward_port') %}
+      Port            ${:forward-port}
+{%- endif %}
+      Shared_Key      ${:forward-shared-key}
+      Self_Hostname   ${:forward-self-hostname}
+      tls             on
+      tls.verify      off
+
+[xlog-fluentbit-service]
+recipe  = slapos.cookbook:wrapper
+fluentbit =  {{ fluent_bit_location }}/bin/fluent-bit
+fluentbit-config = ${xlog-fluentbit-config:output}
+command-line = ${:fluentbit} -c ${:fluentbit-config}
+wrapper-path = ${directory:service}/${:_buildout_section_name_}
+hash-files = ${:fluentbit-config}
+{% endif %}
+
 [amarisoft-stats-template]
 recipe = slapos.recipe.template:jinja2
 extensions = jinja2.ext.do

software/ors-amarisoft/instance-fdd-enb-input-schema.json

@@ -274,6 +274,21 @@
       "description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
       "type": "number",
       "default": 7
+    },
+    "xlog_fluentbit_forward_host": {
+      "title": "Address to Forward Xlog by Fluenbit",
+      "description": "Address of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_port": {
+      "title": "Port to Forward Xlog by Fluentbit",
+      "description": "Optional Port of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_shared_key": {
+      "title": "Shared Key to Forward Xlog by Fluentbit",
+      "description": "Secret Key Shared with Remote Fluentd or Fluentbit Server for Authentication when Forwarding Xlog",
+      "type": "string"
     }
   }
 }

software/ors-amarisoft/instance-fdd-gnb-input-schema.json

@@ -316,6 +316,21 @@
       "description": "Average TX/RX diff threshold above which baseband latency promise will fail",
       "type": "number",
       "default": 7
+    },
+    "xlog_fluentbit_forward_host": {
+      "title": "Address to Forward Xlog by Fluenbit",
+      "description": "Address of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_port": {
+      "title": "Port to Forward Xlog by Fluentbit",
+      "description": "Optional Port of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_shared_key": {
+      "title": "Shared Key to Forward Xlog by Fluentbit",
+      "description": "Secret Key Shared with Remote Fluentd or Fluentbit Server for Authentication when Forwarding Xlog",
+      "type": "string"
     }
   }
 }

software/ors-amarisoft/instance-gnb-input-schema.json.jinja2

@@ -328,6 +328,21 @@
       "description": "Average TX/RX diff threshold above which baseband latency promise will fail",
       "type": "number",
       "default": 7
+    },
+    "xlog_fluentbit_forward_host": {
+      "title": "Address to Forward Xlog by Fluenbit",
+      "description": "Address of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_port": {
+      "title": "Port to Forward Xlog by Fluentbit",
+      "description": "Optional Port of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_shared_key": {
+      "title": "Shared Key to Forward Xlog by Fluentbit",
+      "description": "Secret Key Shared with Remote Fluentd or Fluentbit Server for Authentication when Forwarding Xlog",
+      "type": "string"
     }
   }
 }

software/ors-amarisoft/instance-gnb.jinja2.cfg

@@ -5,6 +5,9 @@ parts =
   gnb-config
   enb-service
   xamari-xlog-service
+{% if slapparameter_dict.get('xlog_fluentbit_forward_host') %}
+  xlog-fluentbit-service
+{% endif %}
   amarisoft-stats-service
   amarisoft-rf-info-service
 {% if slapparameter_dict.get('rrh', '') == "Lopcomm ORAN" %}
@@ -153,6 +156,43 @@ wrapper-path = ${directory:service}/${:_buildout_section_name_}
 command-line = ${xamari-xlog-script:output}
 hash-files = ${:command-line}
 
+{% if slapparameter_dict.get('xlog_fluentbit_forward_host') %}
+[xlog-fluentbit-config]
+recipe = slapos.recipe.template
+output = ${directory:etc}/${:_buildout_section_name_}.cfg
+logfile = ${xamari-xlog-script:logfile}
+forward-host = {{ slapparameter_dict.get('xlog_fluentbit_forward_host', '') }}
+forward-port = {{ slapparameter_dict.get('xlog_fluentbit_forward_port', '') }}
+forward-shared-key = {{ slapparameter_dict.get('xlog_fluentbit_forward_shared_key', '') }}
+forward-self-hostname = {{ ors_id['ors-id'] }}
+inline =
+  [SERVICE]
+      flush           5
+  [INPUT]
+      name            tail
+      path            ${:logfile}
+      Read_from_Head  True
+  [OUTPUT]
+      name            forward
+      match           *
+      Host            ${:forward-host}
+{%- if slapparameter_dict.get('xlog_fluentbit_forward_port') %}
+      Port            ${:forward-port}
+{%- endif %}
+      Shared_Key      ${:forward-shared-key}
+      Self_Hostname   ${:forward-self-hostname}
+      tls             on
+      tls.verify      off
+
+[xlog-fluentbit-service]
+recipe  = slapos.cookbook:wrapper
+fluentbit =  {{ fluent_bit_location }}/bin/fluent-bit
+fluentbit-config = ${xlog-fluentbit-config:output}
+command-line = ${:fluentbit} -c ${:fluentbit-config}
+wrapper-path = ${directory:service}/${:_buildout_section_name_}
+hash-files = ${:fluentbit-config}
+{% endif %}
+
 [amarisoft-stats-template]
 recipe = slapos.recipe.template:jinja2
 extensions = jinja2.ext.do

software/ors-amarisoft/instance-tdd-enb-input-schema.json

@@ -172,6 +172,21 @@
       "description": "Baseband latency promise will fail if average TX/RX diff reaches threshold (higher than this value)",
       "type": "number",
       "default": 7
+    },
+    "xlog_fluentbit_forward_host": {
+      "title": "Address to Forward Xlog by Fluenbit",
+      "description": "Address of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_port": {
+      "title": "Port to Forward Xlog by Fluentbit",
+      "description": "Optional Port of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_shared_key": {
+      "title": "Shared Key to Forward Xlog by Fluentbit",
+      "description": "Secret Key Shared with Remote Fluentd or Fluentbit Server for Authentication when Forwarding Xlog",
+      "type": "string"
     }
   }
 }

software/ors-amarisoft/instance-tdd-gnb-input-schema.json

@@ -326,6 +326,21 @@
       "description": "Average TX/RX diff threshold above which baseband latency promise will fail",
       "type": "number",
       "default": 7
+    },
+    "xlog_fluentbit_forward_host": {
+      "title": "Address to Forward Xlog by Fluenbit",
+      "description": "Address of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_port": {
+      "title": "Port to Forward Xlog by Fluentbit",
+      "description": "Optional Port of Remote Fluentd or Fluentbit Server to Forward Xlog",
+      "type": "string"
+    },
+    "xlog_fluentbit_forward_shared_key": {
+      "title": "Shared Key to Forward Xlog by Fluentbit",
+      "description": "Secret Key Shared with Remote Fluentd or Fluentbit Server for Authentication when Forwarding Xlog",
+      "type": "string"
     }
   }
 }

software/ors-amarisoft/instance.cfg

@@ -171,6 +171,16 @@ init =
               except:
                   options['ipv4'] = "0.0.0.0"
 
+[ors-id]
+recipe = slapos.recipe.build
+computer = $${slap-connection:computer-id}
+title = $${slap-configuration:root-instance-title}
+init =
+  import socket
+  options['hostname'] = socket.gethostname()
+  ors_id = '__'.join(options[x] for x in ('hostname', 'computer', 'title'))
+  options['ors-id'] = ors_id
+
 [switch-softwaretype]
 recipe = slapos.cookbook:switch-softwaretype
 enb = dynamic-template-enb:output
@@ -201,6 +211,7 @@ extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:output}
     section ors_version ors-version
+    section ors_id ors-id
     key enb amarisoft:enb
     key sdr amarisoft:sdr
     raw enb_template   ${enb.jinja2.cfg:target}
@@ -221,6 +232,7 @@ extra-context =
     raw dnsmasq_template   ${dnsmasq.jinja2.cfg:target}
     raw dnsmasq_location ${dnsmasq:location}
     key dnsmasq_config_path dnsmasq-config:output
+    raw fluent_bit_location ${fluent-bit:location}
 
 [dynamic-template-gnb]
 < = jinja2-template-base
@@ -230,6 +242,7 @@ extensions = jinja2.ext.do
 extra-context =
     raw monitor_template ${monitor2-template:output}
     section ors_version ors-version
+    section ors_id ors-id
     key enb amarisoft:enb
     key sdr amarisoft:sdr
     raw gnb_template   ${gnb.jinja2.cfg:target}
@@ -244,6 +257,7 @@ extra-context =
     raw default_n_antenna_ul ${default-params:default-n-antenna-ul}
     raw rf_mode ${rf-mode:rf-mode}
     raw python_path ${python3:location}
+    raw fluent_bit_location ${fluent-bit:location}
 
 [dynamic-template-core-network]
 < = jinja2-template-base

software/ors-amarisoft/software-base.cfg

@@ -13,6 +13,7 @@ extends =
   ../../component/pygolang/buildout.cfg
   ../../component/git/buildout.cfg
   ../../component/dnsmasq/buildout.cfg
+  ../../component/fluent-bit/buildout.cfg
 
 parts +=
   template