Merge branch 'update_fluentd_model_to_include_log_flags' into 'master'

Update Fluentd model to support multiple logs

See merge request gitlab-org/gitlab!29458

app/controllers/clusters/applications_controller.rb

@@ -47,7 +47,7 @@ class Clusters::ApplicationsController < Clusters::BaseController
   end
 
   def cluster_application_params
-    params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :modsecurity_enabled, :modsecurity_mode, :host, :port, :protocol)
+    params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :modsecurity_enabled, :modsecurity_mode, :host, :port, :protocol, :waf_log_enabled, :cilium_log_enabled)
   end
 
   def cluster_application_destroy_params

app/models/clusters/applications/fluentd.rb

@@ -4,6 +4,7 @@ module Clusters
   module Applications
     class Fluentd < ApplicationRecord
       VERSION = '2.4.0'
+      CILIUM_CONTAINER_NAME = 'cilium-monitor'
 
       self.table_name = 'clusters_applications_fluentd'
 
@@ -18,6 +19,8 @@ module Clusters
 
       enum protocol: { tcp: 0, udp: 1 }
 
+      validate :has_at_least_one_log_enabled?
+
       def chart
         'stable/fluentd'
       end
@@ -39,6 +42,12 @@ module Clusters
 
       private
 
+      def has_at_least_one_log_enabled?
+        if !waf_log_enabled && !cilium_log_enabled
+          errors.add(:base, _("At least one logging option is required to be enabled"))
+        end
+      end
+
       def content_values
         YAML.load_file(chart_values_file).deep_merge!(specification)
       end
@@ -62,7 +71,7 @@ module Clusters
           program fluentd
           hostname ${kubernetes_host}
           protocol #{protocol}
-          packet_size 65535
+          packet_size 131072
           <buffer kubernetes_host>
           </buffer>
           <format>
@@ -85,7 +94,7 @@ module Clusters
         <source>
           @type tail
           @id in_tail_container_logs
-          path /var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log
+          path #{path_to_logs}
           pos_file /var/log/fluentd-containers.log.pos
           tag kubernetes.*
           read_from_head true
@@ -96,6 +105,13 @@ module Clusters
         </source>
         EOF
       end
+
+      def path_to_logs
+        path = []
+        path << "/var/log/containers/*#{Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" if waf_log_enabled
+        path << "/var/log/containers/*#{CILIUM_CONTAINER_NAME}*.log" if cilium_log_enabled
+        path.join(',')
+      end
     end
   end
 end

app/serializers/cluster_application_entity.rb

@@ -19,4 +19,6 @@ class ClusterApplicationEntity < Grape::Entity
   expose :host, if: -> (e, _) { e.respond_to?(:host) }
   expose :port, if: -> (e, _) { e.respond_to?(:port) }
   expose :protocol, if: -> (e, _) { e.respond_to?(:protocol) }
+  expose :waf_log_enabled, if: -> (e, _) { e.respond_to?(:waf_log_enabled) }
+  expose :cilium_log_enabled, if: -> (e, _) { e.respond_to?(:cilium_log_enabled) }
 end

app/services/clusters/applications/base_service.rb

@@ -5,6 +5,8 @@ module Clusters
     class BaseService
       InvalidApplicationError = Class.new(StandardError)
 
+      FLUENTD_KNOWN_ATTRS = %i[host protocol port waf_log_enabled cilium_log_enabled].freeze
+
       attr_reader :cluster, :current_user, :params
 
       def initialize(cluster, user, params = {})
@@ -35,17 +37,7 @@ module Clusters
             application.modsecurity_mode = params[:modsecurity_mode] || 0
           end
 
-          if application.has_attribute?(:host)
-            application.host = params[:host]
-          end
-
-          if application.has_attribute?(:protocol)
-            application.protocol = params[:protocol]
-          end
-
-          if application.has_attribute?(:port)
-            application.port = params[:port]
-          end
+          apply_fluentd_related_attributes(application)
 
           if application.respond_to?(:oauth_application)
             application.oauth_application = create_oauth_application(application, request)
@@ -111,6 +103,12 @@ module Clusters
 
         ::Applications::CreateService.new(current_user, oauth_application_params).execute(request)
       end
+
+      def apply_fluentd_related_attributes(application)
+        FLUENTD_KNOWN_ATTRS.each do |attr|
+          application[attr] = params[attr] if application.has_attribute?(attr)
+        end
+      end
     end
   end
 end

changelogs/unreleased/update_fluentd_model_to_include_log_flags.yml

+---
+title: Update Fluentd model to support multiple logs
+merge_request: 29458
+author:
+type: changed

locale/gitlab.pot

@@ -2632,6 +2632,9 @@ msgstr ""
 msgid "At least one approval from a code owner is required to change files matching the respective CODEOWNER rules."
 msgstr ""
 
+msgid "At least one logging option is required to be enabled"
+msgstr ""
+
 msgid "At least one of group_id or project_id must be specified"
 msgstr ""
 

spec/factories/clusters/applications/helm.rb

@@ -142,6 +142,8 @@ FactoryBot.define do
 
     factory :clusters_applications_fluentd, class: 'Clusters::Applications::Fluentd' do
       host { 'example.com' }
+      waf_log_enabled { true }
+      cilium_log_enabled { true }
       cluster factory: %i(cluster with_installed_helm provided_by_gcp)
 
       trait :no_helm_installed do

spec/fixtures/api/schemas/cluster_status.json

@@ -42,6 +42,8 @@
         "host": {"type": ["string", "null"]},
         "port": {"type": ["integer", "514"]},
         "protocol": {"type": ["integer", "0"]},
+        "waf_log_enabled": {"type": ["boolean", "true"]},
+        "cilium_log_enabled": {"type": ["boolean", "true"]},
         "update_available": { "type": ["boolean", "null"] },
         "can_uninstall": { "type": "boolean" },
         "available_domains": {

spec/models/clusters/applications/fluentd_spec.rb

@@ -3,7 +3,9 @@
 require 'spec_helper'
 
 describe Clusters::Applications::Fluentd do
-  let(:fluentd) { create(:clusters_applications_fluentd) }
+  let(:waf_log_enabled) { true }
+  let(:cilium_log_enabled) { true }
+  let(:fluentd) { create(:clusters_applications_fluentd, waf_log_enabled: waf_log_enabled, cilium_log_enabled: cilium_log_enabled) }
 
   include_examples 'cluster application core specs', :clusters_applications_fluentd
   include_examples 'cluster application status specs', :clusters_applications_fluentd
@@ -47,4 +49,36 @@ describe Clusters::Applications::Fluentd do
       expect(values).to include('output.conf', 'general.conf')
     end
   end
+
+  describe '#values' do
+    let(:modsecurity_log_path) { "/var/log/containers/*#{Clusters::Applications::Ingress::MODSECURITY_LOG_CONTAINER_NAME}*.log" }
+    let(:cilium_log_path) { "/var/log/containers/*#{described_class::CILIUM_CONTAINER_NAME}*.log" }
+
+    subject { fluentd.values }
+
+    context 'with both logs variables set to false' do
+      let(:waf_log_enabled) { false }
+      let(:cilium_log_enabled) { false }
+
+      it "raises ActiveRecord::RecordInvalid" do
+        expect {subject}.to raise_error(ActiveRecord::RecordInvalid)
+      end
+    end
+
+    context 'with both logs variables set to true' do
+      it { is_expected.to include("#{modsecurity_log_path},#{cilium_log_path}") }
+    end
+
+    context 'with waf_log_enabled set to true' do
+      let(:cilium_log_enabled) { false }
+
+      it { is_expected.to include(modsecurity_log_path) }
+    end
+
+    context 'with cilium_log_enabled set to true' do
+      let(:waf_log_enabled) { false }
+
+      it { is_expected.to include(cilium_log_path) }
+    end
+  end
 end

spec/serializers/cluster_application_entity_spec.rb

@@ -77,5 +77,17 @@ describe ClusterApplicationEntity do
         expect(subject[:pages_domain]).to eq(id: pages_domain.id, domain: pages_domain.domain)
       end
     end
+
+    context 'for fluentd application' do
+      let(:application) { build(:clusters_applications_fluentd, :installed) }
+
+      it 'includes host, port, protocol and log fields' do
+        expect(subject[:port]).to eq(514)
+        expect(subject[:host]).to eq("example.com")
+        expect(subject[:protocol]).to eq("tcp")
+        expect(subject[:waf_log_enabled]).to be true
+        expect(subject[:cilium_log_enabled]).to be true
+      end
+    end
   end
 end