Merge branch '329944-remove-fluentd-backend' into 'master'

Remove Fluentd related models and services

See merge request gitlab-org/gitlab!63757

app/controllers/clusters/applications_controller.rb

@@ -47,7 +47,7 @@ class Clusters::ApplicationsController < Clusters::BaseController
   end
 
   def cluster_application_params
-    params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :host, :port, :protocol, :cilium_log_enabled)
+    params.permit(:application, :hostname, :pages_domain_id, :email, :stack, :host, :port, :protocol)
   end
 
   def cluster_application_destroy_params

app/models/clusters/applications/fluentd.rb

-# frozen_string_literal: true
-
-module Clusters
-  module Applications
-    class Fluentd < ApplicationRecord
-      VERSION = '2.4.0'
-      CILIUM_CONTAINER_NAME = 'cilium-monitor'
-
-      self.table_name = 'clusters_applications_fluentd'
-
-      include ::Clusters::Concerns::ApplicationCore
-      include ::Clusters::Concerns::ApplicationStatus
-      include ::Clusters::Concerns::ApplicationVersion
-      include ::Clusters::Concerns::ApplicationData
-      include IgnorableColumns
-
-      default_value_for :version, VERSION
-      default_value_for :port, 514
-      default_value_for :protocol, :tcp
-
-      ignore_column :waf_log_enabled, remove_with: '14.2', remove_after: '2021-07-22'
-
-      enum protocol: { tcp: 0, udp: 1 }
-
-      validate :has_at_least_one_log_enabled?
-
-      def chart
-        'fluentd/fluentd'
-      end
-
-      def repository
-        'https://gitlab-org.gitlab.io/cluster-integration/helm-stable-archive'
-      end
-
-      def install_command
-        helm_command_module::InstallCommand.new(
-          name: 'fluentd',
-          repository: repository,
-          version: VERSION,
-          rbac: cluster.platform_kubernetes_rbac?,
-          chart: chart,
-          files: files
-        )
-      end
-
-      def values
-        content_values.to_yaml
-      end
-
-      private
-
-      def has_at_least_one_log_enabled?
-        errors.add(:base, _("At least one logging option is required to be enabled")) unless cilium_log_enabled
-      end
-
-      def content_values
-        YAML.load_file(chart_values_file).deep_merge!(specification)
-      end
-
-      def specification
-        {
-          "configMaps" => {
-            "output.conf" => output_configuration_content,
-            "general.conf" => general_configuration_content
-          }
-        }
-      end
-
-      def output_configuration_content
-        <<~EOF
-        <match kubernetes.**>
-          @type remote_syslog
-          @id out_kube_remote_syslog
-          host #{host}
-          port #{port}
-          program fluentd
-          hostname ${kubernetes_host}
-          protocol #{protocol}
-          packet_size 131072
-          <buffer kubernetes_host>
-          </buffer>
-          <format>
-            @type ltsv
-          </format>
-        </match>
-        EOF
-      end
-
-      def general_configuration_content
-        <<~EOF
-        <match fluent.**>
-          @type null
-        </match>
-        <source>
-          @type http
-          port 9880
-          bind 0.0.0.0
-        </source>
-        <source>
-          @type tail
-          @id in_tail_container_logs
-          path #{path_to_logs}
-          pos_file /var/log/fluentd-containers.log.pos
-          tag kubernetes.*
-          read_from_head true
-          <parse>
-            @type json
-            time_format %Y-%m-%dT%H:%M:%S.%NZ
-          </parse>
-        </source>
-        EOF
-      end
-
-      def path_to_logs
-        path = []
-        path << "/var/log/containers/*#{CILIUM_CONTAINER_NAME}*.log" if cilium_log_enabled
-        path.join(',')
-      end
-    end
-  end
-end

app/models/clusters/cluster.rb

@@ -21,7 +21,6 @@ module Clusters
       Clusters::Applications::Jupyter.application_name => Clusters::Applications::Jupyter,
       Clusters::Applications::Knative.application_name => Clusters::Applications::Knative,
       Clusters::Applications::ElasticStack.application_name => Clusters::Applications::ElasticStack,
-      Clusters::Applications::Fluentd.application_name => Clusters::Applications::Fluentd,
       Clusters::Applications::Cilium.application_name => Clusters::Applications::Cilium
     }.freeze
     DEFAULT_ENVIRONMENT = '*'
@@ -68,7 +67,6 @@ module Clusters
     has_one_cluster_application :jupyter
     has_one_cluster_application :knative
     has_one_cluster_application :elastic_stack
-    has_one_cluster_application :fluentd
     has_one_cluster_application :cilium
 
     has_many :kubernetes_namespaces

app/serializers/cluster_application_entity.rb

@@ -17,5 +17,4 @@ class ClusterApplicationEntity < Grape::Entity
   expose :host, if: -> (e, _) { e.respond_to?(:host) }
   expose :port, if: -> (e, _) { e.respond_to?(:port) }
   expose :protocol, if: -> (e, _) { e.respond_to?(:protocol) }
-  expose :cilium_log_enabled, if: -> (e, _) { e.respond_to?(:cilium_log_enabled) }
 end

app/services/clusters/applications/base_service.rb

@@ -5,8 +5,6 @@ module Clusters
     class BaseService
       InvalidApplicationError = Class.new(StandardError)
 
-      FLUENTD_KNOWN_ATTRS = %i[host protocol port cilium_log_enabled].freeze
-
       attr_reader :cluster, :current_user, :params
 
       def initialize(cluster, user, params = {})
@@ -29,8 +27,6 @@ module Clusters
             application.stack = params[:stack]
           end
 
-          apply_fluentd_related_attributes(application)
-
           if application.respond_to?(:oauth_application)
             application.oauth_application = create_oauth_application(application, request)
           end
@@ -95,12 +91,6 @@ module Clusters
 
         ::Applications::CreateService.new(current_user, oauth_application_params).execute(request)
       end
-
-      def apply_fluentd_related_attributes(application)
-        FLUENTD_KNOWN_ATTRS.each do |attr|
-          application[attr] = params[attr] if application.has_attribute?(attr)
-        end
-      end
     end
   end
 end

locale/gitlab.pot

@@ -4521,9 +4521,6 @@ msgstr ""
 msgid "At least one approval from a code owner is required to change files matching the respective CODEOWNER rules."
 msgstr ""
 
-msgid "At least one logging option is required to be enabled"
-msgstr ""
-
 msgid "At least one of group_id or project_id must be specified"
 msgstr ""
 

spec/factories/clusters/applications/helm.rb

@@ -132,12 +132,6 @@ FactoryBot.define do
       cluster factory: %i(cluster with_installed_helm provided_by_gcp project)
     end
 
-    factory :clusters_applications_fluentd, class: 'Clusters::Applications::Fluentd' do
-      host { 'example.com' }
-      cilium_log_enabled { true }
-      cluster factory: %i(cluster with_installed_helm provided_by_gcp)
-    end
-
     factory :clusters_applications_cilium, class: 'Clusters::Applications::Cilium' do
       cluster factory: %i(cluster with_installed_helm provided_by_gcp)
     end

spec/factories/clusters/clusters.rb

@@ -101,7 +101,6 @@ FactoryBot.define do
       application_jupyter factory: %i(clusters_applications_jupyter installed)
       application_knative factory: %i(clusters_applications_knative installed)
       application_elastic_stack factory: %i(clusters_applications_elastic_stack installed)
-      application_fluentd factory: %i(clusters_applications_fluentd installed)
       application_cilium factory: %i(clusters_applications_cilium installed)
     end
 

spec/fixtures/api/schemas/cluster_status.json

@@ -42,7 +42,6 @@
         "host": {"type": ["string", "null"]},
         "port": {"type": ["integer", "514"]},
         "protocol": {"type": ["integer", "0"]},
-        "cilium_log_enabled": {"type": ["boolean", "true"]},
         "update_available": { "type": ["boolean", "null"] },
         "can_uninstall": { "type": "boolean" },
         "available_domains": {

spec/models/clusters/applications/fluentd_spec.rb

-# frozen_string_literal: true
-
-require 'spec_helper'
-
-RSpec.describe Clusters::Applications::Fluentd do
-  let(:cilium_log_enabled) { true }
-  let(:fluentd) { create(:clusters_applications_fluentd, cilium_log_enabled: cilium_log_enabled) }
-
-  include_examples 'cluster application core specs', :clusters_applications_fluentd
-  include_examples 'cluster application status specs', :clusters_applications_fluentd
-  include_examples 'cluster application version specs', :clusters_applications_fluentd
-  include_examples 'cluster application initial status specs'
-
-  describe '#can_uninstall?' do
-    subject { fluentd.can_uninstall? }
-
-    it { is_expected.to be true }
-  end
-
-  describe '#install_command' do
-    subject { fluentd.install_command }
-
-    it { is_expected.to be_an_instance_of(Gitlab::Kubernetes::Helm::V3::InstallCommand) }
-
-    it 'is initialized with fluentd arguments' do
-      expect(subject.name).to eq('fluentd')
-      expect(subject.chart).to eq('fluentd/fluentd')
-      expect(subject.version).to eq('2.4.0')
-      expect(subject).to be_rbac
-    end
-
-    context 'application failed to install previously' do
-      let(:fluentd) { create(:clusters_applications_fluentd, :errored, version: '0.0.1') }
-
-      it 'is initialized with the locked version' do
-        expect(subject.version).to eq('2.4.0')
-      end
-    end
-  end
-
-  describe '#files' do
-    let(:application) { fluentd }
-    let(:values) { subject[:'values.yaml'] }
-
-    subject { application.files }
-
-    it 'includes fluentd specific keys in the values.yaml file' do
-      expect(values).to include('output.conf', 'general.conf')
-    end
-  end
-
-  describe '#values' do
-    let(:cilium_log_path) { "/var/log/containers/*#{described_class::CILIUM_CONTAINER_NAME}*.log" }
-
-    subject { fluentd.values }
-
-    context 'with cilium_log_enabled set to false' do
-      let(:cilium_log_enabled) { false }
-
-      it "raises ActiveRecord::RecordInvalid" do
-        expect {subject}.to raise_error(ActiveRecord::RecordInvalid)
-      end
-    end
-
-    context 'with cilium_log_enabled set to true' do
-      let(:cilium_log_enabled) { true }
-
-      it { is_expected.to include(cilium_log_path) }
-    end
-  end
-end

spec/serializers/cluster_application_entity_spec.rb

@@ -77,16 +77,5 @@ RSpec.describe ClusterApplicationEntity do
         expect(subject[:pages_domain]).to eq(id: pages_domain.id, domain: pages_domain.domain)
       end
     end
-
-    context 'for fluentd application' do
-      let(:application) { build(:clusters_applications_fluentd, :installed) }
-
-      it 'includes host, port, protocol and log fields' do
-        expect(subject[:port]).to eq(514)
-        expect(subject[:host]).to eq("example.com")
-        expect(subject[:protocol]).to eq("tcp")
-        expect(subject[:cilium_log_enabled]).to be true
-      end
-    end
   end
 end

vendor/fluentd/values.yaml

-plugins:
-  enabled: true
-  pluginsList: ["fluent-plugin-remote_syslog"]
-
-extraVolumes:
-  - name: varlog
-    hostPath:
-      path: /var/log
-  - name: varlibdockercontainers
-    hostPath:
-      path: /var/lib/docker/containers
-
-extraVolumeMounts:
-  - name: varlog
-    mountPath: /var/log
-  - name: varlibdockercontainers
-    mountPath: /var/lib/docker/containers
-    readOnly: true