Commit 2d6ceb67 authored by Zamir Martins Filho's avatar Zamir Martins Filho Committed by Robert Speicher

Update Fluentd model to support multiple logs

In addition to WAF, Cilium logs have been added
        name: varlog
parent f0cfb07c
......@@ -47,7 +47,7 @@ class Clusters::ApplicationsController < Clusters::BaseController
end
def cluster_application_params
params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :modsecurity_enabled, :modsecurity_mode, :host, :port, :protocol)
params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :modsecurity_enabled, :modsecurity_mode, :host, :port, :protocol, :waf_log_enabled, :cilium_log_enabled)
end
def cluster_application_destroy_params
......
......@@ -4,6 +4,7 @@ module Clusters
module Applications
class Fluentd < ApplicationRecord
VERSION = '2.4.0'
CILIUM_CONTAINER_NAME = 'cilium-monitor'
self.table_name = 'clusters_applications_fluentd'
......@@ -18,6 +19,8 @@ module Clusters
enum protocol: { tcp: 0, udp: 1 }
validate :has_at_least_one_log_enabled?
def chart
'stable/fluentd'
end
......@@ -39,6 +42,12 @@ module Clusters
private
def has_at_least_one_log_enabled?
if !waf_log_enabled && !cilium_log_enabled
errors.add(:base, _("At least one logging option is required to be enabled"))
end
end
def content_values
YAML.load_file(chart_values_file).deep_merge!(specification)
end
......@@ -62,7 +71,7 @@ module Clusters
program fluentd
hostname ${kubernetes_host}
protocol #{protocol}
packet_size 65535
packet_size 131072
<buffer kubernetes_host>
</buffer>
<format>
......@@ -85,7 +94,7 @@ module Clusters
<source>
@type tail
@id in_tail_container_logs
path /var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log
path #{path_to_logs}
pos_file /var/log/fluentd-containers.log.pos
tag kubernetes.*
read_from_head true
......@@ -96,6 +105,13 @@ module Clusters
</source>
EOF
end
def path_to_logs
path = []
path << "/var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" if waf_log_enabled
path << "/var/log/containers/*#{CILIUM_CONTAINER_NAME}*.log" if cilium_log_enabled
path.join(',')
end
end
end
end
......@@ -19,4 +19,6 @@ class ClusterApplicationEntity < Grape::Entity
expose :host, if: -> (e, _) { e.respond_to?(:host) }
expose :port, if: -> (e, _) { e.respond_to?(:port) }
expose :protocol, if: -> (e, _) { e.respond_to?(:protocol) }
expose :waf_log_enabled, if: -> (e, _) { e.respond_to?(:waf_log_enabled) }
expose :cilium_log_enabled, if: -> (e, _) { e.respond_to?(:cilium_log_enabled) }
end
......@@ -5,6 +5,8 @@ module Clusters
class BaseService
InvalidApplicationError = Class.new(StandardError)
FLUENTD_KNOWN_ATTRS = %i[host protocol port waf_log_enabled cilium_log_enabled].freeze
attr_reader :cluster, :current_user, :params
def initialize(cluster, user, params = {})
......@@ -35,17 +37,7 @@ module Clusters
application.modsecurity_mode = params[:modsecurity_mode] || 0
end
if application.has_attribute?(:host)
application.host = params[:host]
end
if application.has_attribute?(:protocol)
application.protocol = params[:protocol]
end
if application.has_attribute?(:port)
application.port = params[:port]
end
apply_fluentd_related_attributes(application)
if application.respond_to?(:oauth_application)
application.oauth_application = create_oauth_application(application, request)
......@@ -111,6 +103,12 @@ module Clusters
::Applications::CreateService.new(current_user, oauth_application_params).execute(request)
end
def apply_fluentd_related_attributes(application)
FLUENTD_KNOWN_ATTRS.each do |attr|
application[attr] = params[attr] if application.has_attribute?(attr)
end
end
end
end
end
---
title: Update Fluentd model to support multiple logs
merge_request: 29458
author:
type: changed
......@@ -2632,6 +2632,9 @@ msgstr ""
msgid "At least one approval from a code owner is required to change files matching the respective CODEOWNER rules."
msgstr ""
msgid "At least one logging option is required to be enabled"
msgstr ""
msgid "At least one of group_id or project_id must be specified"
msgstr ""
......
......@@ -142,6 +142,8 @@ FactoryBot.define do
factory :clusters_applications_fluentd, class: 'Clusters::Applications::Fluentd' do
host { 'example.com' }
waf_log_enabled { true }
cilium_log_enabled { true }
cluster factory: %i(cluster with_installed_helm provided_by_gcp)
trait :no_helm_installed do
......
......@@ -42,6 +42,8 @@
"host": {"type": ["string", "null"]},
"port": {"type": ["integer", "514"]},
"protocol": {"type": ["integer", "0"]},
"waf_log_enabled": {"type": ["boolean", "true"]},
"cilium_log_enabled": {"type": ["boolean", "true"]},
"update_available": { "type": ["boolean", "null"] },
"can_uninstall": { "type": "boolean" },
"available_domains": {
......
......@@ -3,7 +3,9 @@
require 'spec_helper'
describe Clusters::Applications::Fluentd do
let(:fluentd) { create(:clusters_applications_fluentd) }
let(:waf_log_enabled) { true }
let(:cilium_log_enabled) { true }
let(:fluentd) { create(:clusters_applications_fluentd, waf_log_enabled: waf_log_enabled, cilium_log_enabled: cilium_log_enabled) }
include_examples 'cluster application core specs', :clusters_applications_fluentd
include_examples 'cluster application status specs', :clusters_applications_fluentd
......@@ -47,4 +49,36 @@ describe Clusters::Applications::Fluentd do
expect(values).to include('output.conf', 'general.conf')
end
end
describe '#values' do
let(:modsecurity_log_path) { "/var/log/containers/*#{Clusters::Applications::Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" }
let(:cilium_log_path) { "/var/log/containers/*#{described_class::CILIUM_CONTAINER_NAME}*.log" }
subject { fluentd.values }
context 'with both logs variables set to false' do
let(:waf_log_enabled) { false }
let(:cilium_log_enabled) { false }
it "raises ActiveRecord::RecordInvalid" do
expect {subject}.to raise_error(ActiveRecord::RecordInvalid)
end
end
context 'with both logs variables set to true' do
it { is_expected.to include("#{modsecurity_log_path},#{cilium_log_path}") }
end
context 'with waf_log_enabled set to true' do
let(:cilium_log_enabled) { false }
it { is_expected.to include(modsecurity_log_path) }
end
context 'with cilium_log_enabled set to true' do
let(:waf_log_enabled) { false }
it { is_expected.to include(cilium_log_path) }
end
end
end
......@@ -77,5 +77,17 @@ describe ClusterApplicationEntity do
expect(subject[:pages_domain]).to eq(id: pages_domain.id, domain: pages_domain.domain)
end
end
context 'for fluentd application' do
let(:application) { build(:clusters_applications_fluentd, :installed) }
it 'includes host, port, protocol and log fields' do
expect(subject[:port]).to eq(514)
expect(subject[:host]).to eq("example.com")
expect(subject[:protocol]).to eq("tcp")
expect(subject[:waf_log_enabled]).to be true
expect(subject[:cilium_log_enabled]).to be true
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment