Commit d36ebb34 authored by Hordur Freyr Yngvason's avatar Hordur Freyr Yngvason Committed by Thong Kuah

Add Elastic Stack cluster integration

parent 608211ed
......@@ -62,6 +62,7 @@ class Clusters::ClustersController < Clusters::BaseController
def show
if params[:tab] == 'integrations'
@prometheus_integration = Clusters::IntegrationPresenter.new(@cluster.find_or_build_integration_prometheus)
@elastic_stack_integration = Clusters::IntegrationPresenter.new(@cluster.find_or_build_integration_elastic_stack)
end
end
......
......@@ -24,7 +24,7 @@ module Clusters
end
def cluster_integration_params
params.require(:integration).permit(:application_type, :enabled)
params.permit(integration: [:enabled, :application_type]).require(:integration)
end
def cluster
......
......@@ -3,9 +3,9 @@
module Clusters
module Applications
class ElasticStack < ApplicationRecord
VERSION = '3.0.0'
include ::Clusters::Concerns::ElasticsearchClient
ELASTICSEARCH_PORT = 9200
VERSION = '3.0.0'
self.table_name = 'clusters_applications_elastic_stacks'
......@@ -13,10 +13,23 @@ module Clusters
include ::Clusters::Concerns::ApplicationStatus
include ::Clusters::Concerns::ApplicationVersion
include ::Clusters::Concerns::ApplicationData
include ::Gitlab::Utils::StrongMemoize
default_value_for :version, VERSION
after_destroy do
cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil)
end
state_machine :status do
after_transition any => [:installed] do |application|
application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: true, chart_version: application.version)
end
after_transition any => [:uninstalled] do |application|
application.cluster&.find_or_build_integration_elastic_stack&.update(enabled: false, chart_version: nil)
end
end
def chart
'elastic-stack/elastic-stack'
end
......@@ -51,31 +64,6 @@ module Clusters
super.merge('wait-for-elasticsearch.sh': File.read("#{Rails.root}/vendor/elastic_stack/wait-for-elasticsearch.sh"))
end
def elasticsearch_client(timeout: nil)
strong_memoize(:elasticsearch_client) do
next unless kube_client
proxy_url = kube_client.proxy_url('service', service_name, ::Clusters::Applications::ElasticStack::ELASTICSEARCH_PORT, Gitlab::Kubernetes::Helm::NAMESPACE)
Elasticsearch::Client.new(url: proxy_url) do |faraday|
# ensures headers containing auth data are appended to original client options
faraday.headers.merge!(kube_client.headers)
# ensure TLS certs are properly verified
faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl]
faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store]
faraday.options.timeout = timeout unless timeout.nil?
end
rescue Kubeclient::HttpError => error
# If users have mistakenly set parameters or removed the depended clusters,
# `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
# We check for a nil client in downstream use and behaviour is equivalent to an empty state
log_exception(error, :failed_to_create_elasticsearch_client)
nil
end
end
def chart_above_v2?
Gem::Version.new(version) >= Gem::Version.new('2.0.0')
end
......@@ -106,10 +94,6 @@ module Clusters
]
end
def kube_client
cluster&.kubeclient&.core_client
end
def migrate_to_3_script
return [] if !updating? || chart_above_v3?
......
......@@ -52,6 +52,7 @@ module Clusters
has_one :platform_kubernetes, class_name: 'Clusters::Platforms::Kubernetes', inverse_of: :cluster, autosave: true
has_one :integration_prometheus, class_name: 'Clusters::Integrations::Prometheus', inverse_of: :cluster
has_one :integration_elastic_stack, class_name: 'Clusters::Integrations::ElasticStack', inverse_of: :cluster
def self.has_one_cluster_application(name) # rubocop:disable Naming/PredicateName
application = APPLICATIONS[name.to_s]
......@@ -104,6 +105,7 @@ module Clusters
delegate :available?, to: :application_ingress, prefix: true, allow_nil: true
delegate :available?, to: :application_knative, prefix: true, allow_nil: true
delegate :available?, to: :application_elastic_stack, prefix: true, allow_nil: true
delegate :available?, to: :integration_elastic_stack, prefix: true, allow_nil: true
delegate :external_ip, to: :application_ingress, prefix: true, allow_nil: true
delegate :external_hostname, to: :application_ingress, prefix: true, allow_nil: true
......@@ -284,6 +286,10 @@ module Clusters
integration_prometheus || build_integration_prometheus
end
def find_or_build_integration_elastic_stack
integration_elastic_stack || build_integration_elastic_stack
end
def provider
if gcp?
provider_gcp
......@@ -318,6 +324,22 @@ module Clusters
platform_kubernetes.kubeclient if kubernetes?
end
def elastic_stack_adapter
application_elastic_stack || integration_elastic_stack
end
def elasticsearch_client
elastic_stack_adapter&.elasticsearch_client
end
def elastic_stack_available?
if application_elastic_stack_available? || integration_elastic_stack_available?
true
else
false
end
end
def kubernetes_namespace_for(environment, deployable: environment.last_deployable)
if deployable && environment.project_id != deployable.project_id
raise ArgumentError, 'environment.project_id must match deployable.project_id'
......
......@@ -6,6 +6,8 @@ module Clusters
extend ActiveSupport::Concern
included do
include ::Clusters::Concerns::KubernetesLogger
belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
validates :cluster, presence: true
......@@ -79,24 +81,6 @@ module Clusters
# Override if your application needs any action after
# being uninstalled by Helm
end
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def log_exception(error, event)
logger.error({
exception: error.class.name,
status_code: error.error_code,
cluster_id: cluster&.id,
application_id: id,
class_name: self.class.name,
event: event,
message: error.message
})
Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
end
end
end
end
......
# frozen_string_literal: true
module Clusters
module Concerns
module ElasticsearchClient
include ::Gitlab::Utils::StrongMemoize
ELASTICSEARCH_PORT = 9200
ELASTICSEARCH_NAMESPACE = 'gitlab-managed-apps'
def elasticsearch_client(timeout: nil)
strong_memoize(:elasticsearch_client) do
kube_client = cluster&.kubeclient&.core_client
next unless kube_client
proxy_url = kube_client.proxy_url('service', service_name, ELASTICSEARCH_PORT, ELASTICSEARCH_NAMESPACE)
Elasticsearch::Client.new(url: proxy_url) do |faraday|
# ensures headers containing auth data are appended to original client options
faraday.headers.merge!(kube_client.headers)
# ensure TLS certs are properly verified
faraday.ssl[:verify] = kube_client.ssl_options[:verify_ssl]
faraday.ssl[:cert_store] = kube_client.ssl_options[:cert_store]
faraday.options.timeout = timeout unless timeout.nil?
end
rescue Kubeclient::HttpError => error
# If users have mistakenly set parameters or removed the depended clusters,
# `proxy_url` could raise an exception because gitlab can not communicate with the cluster.
# We check for a nil client in downstream use and behaviour is equivalent to an empty state
log_exception(error, :failed_to_create_elasticsearch_client)
nil
end
end
end
end
end
# frozen_string_literal: true
module Clusters
module Concerns
module KubernetesLogger
def logger
@logger ||= Gitlab::Kubernetes::Logger.build
end
def log_exception(error, event)
logger.error(
{
exception: error.class.name,
status_code: error.error_code,
cluster_id: cluster&.id,
application_id: id,
class_name: self.class.name,
event: event,
message: error.message
}
)
Gitlab::ErrorTracking.track_exception(error, cluster_id: cluster&.id, application_id: id)
end
end
end
end
# frozen_string_literal: true
module Clusters
module Integrations
class ElasticStack < ApplicationRecord
include ::Clusters::Concerns::ElasticsearchClient
include ::Clusters::Concerns::KubernetesLogger
self.table_name = 'clusters_integration_elasticstack'
self.primary_key = :cluster_id
belongs_to :cluster, class_name: 'Clusters::Cluster', foreign_key: :cluster_id
validates :cluster, presence: true
validates :enabled, inclusion: { in: [true, false] }
def available?
enabled
end
def service_name
chart_above_v3? ? 'elastic-stack-elasticsearch-master' : 'elastic-stack-elasticsearch-client'
end
def chart_above_v2?
return true if chart_version.nil?
Gem::Version.new(chart_version) >= Gem::Version.new('2.0.0')
end
def chart_above_v3?
return true if chart_version.nil?
Gem::Version.new(chart_version) >= Gem::Version.new('3.0.0')
end
end
end
end
......@@ -406,7 +406,7 @@ class Environment < ApplicationRecord
end
def elastic_stack_available?
!!deployment_platform&.cluster&.application_elastic_stack_available?
!!deployment_platform&.cluster&.elastic_stack_available?
end
def rollout_status
......
......@@ -76,7 +76,7 @@ module Clusters
def gitlab_managed_apps_logs_path
return unless logs_project && can_read_cluster?
if cluster.application_elastic_stack&.available?
if cluster.elastic_stack_adapter&.available?
elasticsearch_project_logs_path(logs_project, cluster_id: cluster.id, format: :json)
else
k8s_project_logs_path(logs_project, cluster_id: cluster.id, format: :json)
......
......@@ -28,6 +28,6 @@ class ClusterEntity < Grape::Entity
end
expose :enable_advanced_logs_querying do |cluster|
cluster.application_elastic_stack_available?
cluster.elastic_stack_available?
end
end
......@@ -27,12 +27,15 @@ module Clusters
private
def integration
case params[:application_type]
when 'prometheus'
cluster.find_or_build_integration_prometheus
else
raise ArgumentError, "invalid application_type: #{params[:application_type]}"
end
@integration ||= \
case params[:application_type]
when 'prometheus'
cluster.find_or_build_integration_prometheus
when 'elastic_stack'
cluster.find_or_build_integration_elastic_stack
else
raise ArgumentError, "invalid application_type: #{params[:application_type]}"
end
end
def authorized?
......
......@@ -24,7 +24,7 @@ module PodLogs
end
def get_raw_pods(result)
client = cluster&.application_elastic_stack&.elasticsearch_client
client = cluster&.elasticsearch_client
return error(_('Unable to connect to Elasticsearch')) unless client
result[:raw_pods] = ::Gitlab::Elasticsearch::Logs::Pods.new(client).pods(namespace)
......@@ -66,11 +66,9 @@ module PodLogs
end
def pod_logs(result)
client = cluster&.application_elastic_stack&.elasticsearch_client
client = cluster&.elasticsearch_client
return error(_('Unable to connect to Elasticsearch')) unless client
chart_above_v2 = cluster.application_elastic_stack.chart_above_v2?
response = ::Gitlab::Elasticsearch::Logs::Lines.new(client).pod_logs(
namespace,
pod_name: result[:pod_name],
......@@ -79,7 +77,7 @@ module PodLogs
start_time: result[:start_time],
end_time: result[:end_time],
cursor: result[:cursor],
chart_above_v2: chart_above_v2
chart_above_v2: cluster.elastic_stack_adapter.chart_above_v2?
)
result.merge!(response)
......
.settings.expanded.border-0.m-0
%p
= s_('ClusterIntegration|Integrations enable you to integrate your cluster as part of your GitLab workflow.')
= s_('ClusterIntegration|Integrations allow you to use applications installed in your cluster as part of your GitLab workflow.')
= link_to _('Learn more'), help_page_path('user/clusters/integrations.md'), target: '_blank'
.settings-content#advanced-settings-section
.settings-content#integrations-settings-section
- if can?(current_user, :admin_cluster, @cluster)
.sub-section.form-group
= form_for @prometheus_integration, url: @cluster.integrations_path, as: :integration, method: :post, html: { class: 'js-cluster-integrations-form' } do |form|
= form.hidden_field :application_type
.form-group
= form_for @prometheus_integration, as: :integration, namespace: :prometheus, url: @cluster.integrations_path, method: :post, html: { class: 'js-cluster-integrations-form' } do |prometheus_form|
= prometheus_form.hidden_field :application_type
.form-group.gl-form-group
.gl-form-checkbox.custom-control.custom-checkbox
= form.check_box :enabled, { class: 'custom-control-input'}
= form.label :enabled, s_('ClusterIntegration|Enable Prometheus integration'), class: 'custom-control-label'
.gl-form-group
= prometheus_form.check_box :enabled, class: 'custom-control-input'
= prometheus_form.label :enabled, s_('ClusterIntegration|Enable Prometheus integration'), class: 'custom-control-label'
.form-text.text-gl-muted
- link_start = '<a href="%{url}" target="_blank" rel="noopener noreferrer">'.html_safe % { url: help_page_path("user/clusters/integrations", anchor: "prometheus-cluster-integration") }
- link_end = '</a>'.html_safe
= html_escape(s_('ClusterIntegration|Before you enable this integration, follow the %{link_start}documented process%{link_end}.')) % { link_start: link_start, link_end: link_end }
= form.submit _('Save changes'), class: 'btn gl-button btn-success'
= s_('ClusterIntegration|Allows GitLab to query a specifically configured in-cluster Prometheus for metrics.')
= link_to _('More information.'), help_page_path("user/clusters/integrations", anchor: "prometheus-cluster-integration"), target: '_blank'
= prometheus_form.submit _('Save changes'), class: 'btn gl-button btn-success'
.sub-section.form-group
= form_for @elastic_stack_integration, as: :integration, namespace: :elastic_stack, url: @cluster.integrations_path, method: :post, html: { class: 'js-cluster-integrations-form' } do |elastic_stack_form|
= elastic_stack_form.hidden_field :application_type
.form-group.gl-form-group
.gl-form-checkbox.custom-control.custom-checkbox
= elastic_stack_form.check_box :enabled, class: 'custom-control-input'
= elastic_stack_form.label :enabled, s_('ClusterIntegration|Enable Elastic Stack integration'), class: 'custom-control-label'
.form-text.text-gl-muted
= s_('ClusterIntegration|Allows GitLab to query a specifically configured in-cluster Elasticsearch for pod logs.')
= link_to _('More information.'), help_page_path("user/clusters/integrations", anchor: "elastic-stack-cluster-integration"), target: '_blank'
= elastic_stack_form.submit _('Save changes'), class: 'btn gl-button btn-success'
---
title: Add Elastic Stack cluster integration
merge_request: 61077
author:
type: added
# frozen_string_literal: true
class CreateClustersIntegrationElasticstack < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
def change
create_table_with_constraints :clusters_integration_elasticstack, id: false do |t|
t.timestamps_with_timezone null: false
t.references :cluster, primary_key: true, default: nil, index: false, foreign_key: { on_delete: :cascade }
t.boolean :enabled, null: false, default: false
t.text :chart_version
t.text_limit :chart_version, 10
end
end
end
c4593c1638f937618ecf3ae94a409e550dce93cc190989f581fb0007e591696d
\ No newline at end of file
......@@ -11715,6 +11715,15 @@ CREATE SEQUENCE clusters_id_seq
ALTER SEQUENCE clusters_id_seq OWNED BY clusters.id;
CREATE TABLE clusters_integration_elasticstack (
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
cluster_id bigint NOT NULL,
enabled boolean DEFAULT false NOT NULL,
chart_version text,
CONSTRAINT check_f8d671ce04 CHECK ((char_length(chart_version) <= 10))
);
CREATE TABLE clusters_integration_prometheus (
created_at timestamp with time zone NOT NULL,
updated_at timestamp with time zone NOT NULL,
......@@ -20734,6 +20743,9 @@ ALTER TABLE ONLY clusters_applications_prometheus
ALTER TABLE ONLY clusters_applications_runners
ADD CONSTRAINT clusters_applications_runners_pkey PRIMARY KEY (id);
ALTER TABLE ONLY clusters_integration_elasticstack
ADD CONSTRAINT clusters_integration_elasticstack_pkey PRIMARY KEY (cluster_id);
ALTER TABLE ONLY clusters_integration_prometheus
ADD CONSTRAINT clusters_integration_prometheus_pkey PRIMARY KEY (cluster_id);
......@@ -26969,6 +26981,9 @@ ALTER TABLE ONLY boards_epic_board_positions
ALTER TABLE ONLY vulnerability_finding_links
ADD CONSTRAINT fk_rails_cbdfde27ce FOREIGN KEY (vulnerability_occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;
ALTER TABLE ONLY clusters_integration_elasticstack
ADD CONSTRAINT fk_rails_cc5ba8f658 FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE;
ALTER TABLE ONLY issues_self_managed_prometheus_alert_events
ADD CONSTRAINT fk_rails_cc5d88bbb0 FOREIGN KEY (issue_id) REFERENCES issues(id) ON DELETE CASCADE;
......@@ -10,7 +10,9 @@ GitLab provides several ways to integrate applications to your
Kubernetes cluster.
To enable cluster integrations, first add a Kubernetes cluster to a GitLab
[project](../project/clusters/add_remove_clusters.md) or [group](../group/clusters/index.md#group-level-kubernetes-clusters).
[project](../project/clusters/add_remove_clusters.md) or
[group](../group/clusters/index.md#group-level-kubernetes-clusters) or
[instance](../instance/clusters/index.md).
## Prometheus cluster integration
......@@ -20,33 +22,33 @@ You can integrate your Kubernetes cluster with
[Prometheus](https://prometheus.io/) for monitoring key metrics of your
apps directly from the GitLab UI.
[Alerts](../../operations/metrics/alerts.md) are not currently
supported.
[Alerts](../../operations/metrics/alerts.md) can be configured the same way as
for [external Prometheus instances](../../operations/metrics/alerts.md#external-prometheus-instances).
Once enabled, you will see metrics from services available in the
Once enabled, you can see metrics from services available in the
[metrics library](../project/integrations/prometheus_library/index.md).
Prerequisites:
### Prometheus Prerequisites
To benefit from this integration, you must have Prometheus
installed in your cluster with the following requirements:
To use this integration:
1. Prometheus must be installed inside the `gitlab-managed-apps` namespace.
1. Prometheus must be installed in your cluster in the `gitlab-managed-apps` namespace.
1. The `Service` resource for Prometheus must be named `prometheus-prometheus-server`.
You can use the following commands to install Prometheus to meet the requirements for cluster integrations:
You can manage your Prometheus however you like, but as an example, you can set
it up using [Helm](https://helm.sh/) as follows:
```shell
# Create the require Kubernetes namespace
# Create the required Kubernetes namespace
kubectl create ns gitlab-managed-apps
# Download Helm chart values that is compatible with the requirements above.
# You should substitute the tag that corresponds to the GitLab version in the url
# You should substitute the tag that corresponds to the GitLab version in the URL
# - https://gitlab.com/gitlab-org/gitlab/-/raw/<tag>/vendor/prometheus/values.yaml
#
wget https://gitlab.com/gitlab-org/gitlab/-/raw/v13.9.0-ee/vendor/prometheus/values.yaml
# Add the Prometheus community helm repo
# Add the Prometheus community Helm chart repository
helm repo add prometheus-community https://prometheus-community.github.io/helm-charts
# Install Prometheus
......@@ -65,6 +67,65 @@ To enable the Prometheus integration for your cluster:
**Operations > Kubernetes**.
- For a [group-level cluster](../group/clusters/index.md), navigate to your group's
**Kubernetes** page.
- For an [instance-level cluster](../instance/clusters/index.md), navigate to your instance's
**Kubernetes** page.
1. Select the **Integrations** tab.
1. Check the **Enable Prometheus integration** checkbox.
1. Click **Save changes**.
1. Go to the **Health** tab to see your cluster's metrics.
## Elastic Stack cluster integration
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/merge_requests/61077) in GitLab 13.12.
You can integrate your cluster with [Elastic
Stack](https://www.elastic.co/elastic-stack) to index and [query your pod
logs](../project/clusters/kubernetes_pod_logs.md).
### Elastic Stack Prerequisites
To use this integration:
1. Elasticsearch 7.x or must be installed in your cluster in the
`gitlab-managed-apps` namespace.
1. The `Service` resource must be called `elastic-stack-elasticsearch-master`
and expose the Elasticsearch API on port `9200`.
1. The logs are expected to be [Filebeat container logs](https://www.elastic.co/guide/en/beats/filebeat/7.x/filebeat-input-container.html)
following the [7.x log structure](https://www.elastic.co/guide/en/beats/filebeat/7.x/exported-fields-log.html)
and include [Kubernetes metadata](https://www.elastic.co/guide/en/beats/filebeat/7.x/add-kubernetes-metadata.html).
You can manage your Elastic Stack however you like, but as an example, you can
use [this Elastic Stack chart](https://gitlab.com/gitlab-org/charts/elastic-stack) to get up and
running:
```shell
# Create the required Kubernetes namespace
kubectl create namespace gitlab-managed-apps
# Download Helm chart values that is compatible with the requirements above.
# You should substitute the tag that corresponds to the GitLab version in the URL
# - https://gitlab.com/gitlab-org/gitlab/-/raw/<tag>/vendor/elastic_stack/values.yaml
#
wget https://gitlab.com/gitlab-org/gitlab/-/raw/v13.9.0-ee/vendor/elastic_stack/values.yaml
# Add the GitLab Helm chart repository
helm repo add gitlab https://charts.gitlab.io
# Install Elastic Stack
helm install prometheus gitlab/elastic-stack -n gitlab-managed-apps --values values.yaml
```
### Enable Elastic Stack integration for your cluster
To enable the Elastic Stack integration for your cluster:
1. Go to the cluster's page:
- For a [project-level cluster](../project/clusters/index.md), navigate to your project's
**Operations > Kubernetes**.
- For a [group-level cluster](../group/clusters/index.md), navigate to your group's
**Kubernetes** page.
- For an [instance-level cluster](../instance/clusters/index.md), navigate to your instance's
**Kubernetes** page.
1. Select the **Integrations** tab.
1. Check the **Enable Prometheus integration** checkbox.
1. Click **Save changes**.
......
......@@ -50,17 +50,17 @@ module Security
end
def elasticsearch_client
@elasticsearch_client ||= application_elastic_stack&.elasticsearch_client(timeout: @options[:timeout])
@elasticsearch_client ||= elastic_stack_adapter&.elasticsearch_client(timeout: @options[:timeout])
end
private
def application_elastic_stack
@application_elastic_stack ||= @cluster&.application_elastic_stack
def elastic_stack_adapter
@elastic_stack_adapter ||= @cluster&.elastic_stack_adapter
end
def chart_above_v3?
application_elastic_stack.chart_above_v3?
elastic_stack_adapter.chart_above_v3?
end
def body
......
......@@ -6932,6 +6932,12 @@ msgstr ""
msgid "ClusterIntegration|Allow GitLab to manage namespaces and service accounts for this cluster."
msgstr ""
msgid "ClusterIntegration|Allows GitLab to query a specifically configured in-cluster Elasticsearch for pod logs."
msgstr ""
msgid "ClusterIntegration|Allows GitLab to query a specifically configured in-cluster Prometheus for metrics."
msgstr ""
msgid "ClusterIntegration|Alternatively, "
msgstr ""
......@@ -6974,9 +6980,6 @@ msgstr ""
msgid "ClusterIntegration|Base domain"
msgstr ""
msgid "ClusterIntegration|Before you enable this integration, follow the %{link_start}documented process%{link_end}."
msgstr ""
msgid "ClusterIntegration|Blocking mode"
msgstr ""
......@@ -7142,6 +7145,9 @@ msgstr ""
msgid "ClusterIntegration|Enable Cloud Run for Anthos"
msgstr ""
msgid "ClusterIntegration|Enable Elastic Stack integration"
msgstr ""
msgid "ClusterIntegration|Enable Prometheus integration"
msgstr ""
......@@ -7289,7 +7295,7 @@ msgstr ""
msgid "ClusterIntegration|Integration enabled"
msgstr ""
msgid "ClusterIntegration|Integrations enable you to integrate your cluster as part of your GitLab workflow."
msgid "ClusterIntegration|Integrations allow you to use applications installed in your cluster as part of your GitLab workflow."
msgstr ""
msgid "ClusterIntegration|Issuer Email"
......
# frozen_string_literal: true
FactoryBot.define do
factory :clusters_integrations_elastic_stack, class: 'Clusters::Integrations::ElasticStack' do
cluster factory: %i(cluster provided_by_gcp)
enabled { true }
trait :disabled do
enabled { false }
end
end
end
......@@ -10,6 +10,41 @@ RSpec.describe Clusters::Applications::ElasticStack do
include_examples 'cluster application version specs', :clusters_applications_elastic_stack
include_examples 'cluster application helm specs', :clusters_applications_elastic_stack
describe 'cluster.integration_elastic_stack state synchronization' do
let!(:application) { create(:clusters_applications_elastic_stack) }
let(:cluster) { application.cluster }
let(:integration) { cluster.integration_elastic_stack }
describe 'after_destroy' do
it 'disables the corresponding integration' do
application.destroy!
expect(integration).not_to be_enabled
end
end
describe 'on install' do
it 'enables the corresponding integration' do
application.make_scheduled!
application.make_installing!
application.make_installed!
expect(integration).to be_enabled
end
end
describe 'on uninstall' do
it 'disables the corresponding integration' do
application.make_scheduled!
application.make_installing!
application.make_installed!
application.make_externally_uninstalled!
expect(integration).not_to be_enabled
end
end
end
describe '#install_command' do
let!(:elastic_stack) { create(:clusters_applications_elastic_stack) }
......@@ -138,78 +173,5 @@ RSpec.describe Clusters::Applications::ElasticStack do
end
end
describe '#elasticsearch_client' do
context 'cluster is nil' do
it 'returns nil' do
expect(subject.cluster).to be_nil
expect(subject.elasticsearch_client).to be_nil
end
end
context "cluster doesn't have kubeclient" do
let(:cluster) { create(:cluster) }
subject { create(:clusters_applications_elastic_stack, cluster: cluster) }
it 'returns nil' do
expect(subject.elasticsearch_client).to be_nil
end
end
context 'cluster has kubeclient' do
let(:cluster) { create(:cluster, :project, :provided_by_gcp) }
let(:kubernetes_url) { subject.cluster.platform_kubernetes.api_url }
let(:kube_client) { subject.cluster.kubeclient.core_client }
subject { create(:clusters_applications_elastic_stack, cluster: cluster) }
before do
subject.cluster.platform_kubernetes.namespace = 'a-namespace'
stub_kubeclient_discover(cluster.platform_kubernetes.api_url)
create(:cluster_kubernetes_namespace,
cluster: cluster,
cluster_project: cluster.cluster_project,
project: cluster.cluster_project.project)
end
it 'creates proxy elasticsearch_client' do
expect(subject.elasticsearch_client).to be_instance_of(Elasticsearch::Transport::Client)
end
it 'copies proxy_url, options and headers from kube client to elasticsearch_client' do
expect(Elasticsearch::Client)
.to(receive(:new))
.with(url: a_valid_url)
.and_call_original
client = subject.elasticsearch_client
faraday_connection = client.transport.connections.first.connection
expect(faraday_connection.headers["Authorization"]).to eq(kube_client.headers[:Authorization])
expect(faraday_connection.ssl.cert_store).to be_instance_of(OpenSSL::X509::Store)
expect(faraday_connection.ssl.verify).to eq(1)
expect(faraday_connection.options.timeout).to be_nil
end
context 'when cluster is not reachable' do
before do
allow(kube_client).to receive(:proxy_url).and_raise(Kubeclient::HttpError.new(401, 'Unauthorized', nil))
end
it 'returns nil' do
expect(subject.elasticsearch_client).to be_nil
end
end
context 'when timeout is provided' do
it 'sets timeout in elasticsearch_client' do
client = subject.elasticsearch_client(timeout: 123)
faraday_connection = client.transport.connections.first.connection
expect(faraday_connection.options.timeout).to eq(123)
end
end
end
end
it_behaves_like 'cluster-based #elasticsearch_client', :clusters_applications_elastic_stack
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Clusters::Integrations::ElasticStack do
include KubernetesHelpers
include StubRequests
describe 'associations' do
it { is_expected.to belong_to(:cluster).class_name('Clusters::Cluster') }
end
describe 'validations' do
it { is_expected.to validate_presence_of(:cluster) }
it { is_expected.not_to allow_value(nil).for(:enabled) }
end
it_behaves_like 'cluster-based #elasticsearch_client', :clusters_integrations_elastic_stack
end
......@@ -6,79 +6,64 @@ RSpec.describe Clusters::Integrations::CreateService, '#execute' do
let_it_be(:project) { create(:project) }
let_it_be_with_reload(:cluster) { create(:cluster, :provided_by_gcp, projects: [project]) }
let(:params) do
{ application_type: 'prometheus', enabled: true }
end
let(:service) do
described_class.new(container: project, cluster: cluster, current_user: project.owner, params: params)
end
it 'creates a new Prometheus instance' do
expect(service.execute).to be_success
expect(cluster.integration_prometheus).to be_present
expect(cluster.integration_prometheus).to be_persisted
expect(cluster.integration_prometheus).to be_enabled
end
context 'enabled param is false' do
let(:params) do
{ application_type: 'prometheus', enabled: false }
end
it 'creates a new uninstalled Prometheus instance' do
expect(service.execute).to be_success
shared_examples_for 'a cluster integration' do |application_type|
let(:integration) { cluster.public_send("integration_#{application_type}") }
expect(cluster.integration_prometheus).to be_present
expect(cluster.integration_prometheus).to be_persisted
expect(cluster.integration_prometheus).not_to be_enabled
end
end
context 'when enabled param is true' do
let(:params) do
{ application_type: application_type, enabled: true }
end
context 'unauthorized user' do
let(:service) do
unauthorized_user = create(:user)
it 'creates a new enabled integration' do
expect(service.execute).to be_success
described_class.new(container: project, cluster: cluster, current_user: unauthorized_user, params: params)
expect(integration).to be_present
expect(integration).to be_persisted
expect(integration).to be_enabled
end
end
it 'does not create a new Prometheus instance' do
expect(service.execute).to be_error
context 'when enabled param is false' do
let(:params) do
{ application_type: application_type, enabled: false }
end
expect(cluster.integration_prometheus).to be_nil
end
end
it 'creates a new disabled integration' do
expect(service.execute).to be_success
context 'prometheus record exists' do
before do
create(:clusters_integrations_prometheus, cluster: cluster)
expect(integration).to be_present
expect(integration).to be_persisted
expect(integration).not_to be_enabled
end
end
it 'updates the Prometheus instance' do
expect(service.execute).to be_success
expect(cluster.integration_prometheus).to be_present
expect(cluster.integration_prometheus).to be_persisted
expect(cluster.integration_prometheus).to be_enabled
end
context 'when integration already exists' do
before do
create(:"clusters_integrations_#{application_type}", cluster: cluster, enabled: false)
end
context 'enabled param is false' do
let(:params) do
{ application_type: 'prometheus', enabled: false }
{ application_type: application_type, enabled: true }
end
it 'updates the Prometheus instance as uninstalled' do
it 'updates the integration' do
expect(integration).not_to be_enabled
expect(service.execute).to be_success
expect(cluster.integration_prometheus).to be_present
expect(cluster.integration_prometheus).to be_persisted
expect(cluster.integration_prometheus).not_to be_enabled
expect(integration.reload).to be_enabled
end
end
end
context 'for an un-supported application type' do
it_behaves_like 'a cluster integration', 'prometheus'
it_behaves_like 'a cluster integration', 'elastic_stack'
context 'when application_type is invalid' do
let(:params) do
{ application_type: 'something_else', enabled: true }
end
......@@ -87,4 +72,22 @@ RSpec.describe Clusters::Integrations::CreateService, '#execute' do
expect { service.execute}.to raise_error(ArgumentError)
end
end
context 'when user is unauthorized' do
let(:params) do
{ application_type: 'prometheus', enabled: true }
end
let(:service) do
unauthorized_user = create(:user)
described_class.new(container: project, cluster: cluster, current_user: unauthorized_user, params: params)
end
it 'returns error and does not create a new integration record' do
expect(service.execute).to be_error
expect(cluster.integration_prometheus).to be_nil
end
end
end
# frozen_string_literal: true
# Input
# - factory: [:clusters_applications_elastic_stack, :clusters_integrations_elastic_stack]
RSpec.shared_examples 'cluster-based #elasticsearch_client' do |factory|
describe '#elasticsearch_client' do
context 'cluster is nil' do
subject { build(factory, cluster: nil) }
it 'returns nil' do
expect(subject.cluster).to be_nil
expect(subject.elasticsearch_client).to be_nil
end
end
context "cluster doesn't have kubeclient" do
let(:cluster) { create(:cluster) }
subject { create(factory, cluster: cluster) }
it 'returns nil' do
expect(subject.elasticsearch_client).to be_nil
end
end
context 'cluster has kubeclient' do
let(:cluster) { create(:cluster, :project, :provided_by_gcp) }
let(:kubernetes_url) { subject.cluster.platform_kubernetes.api_url }
let(:kube_client) { subject.cluster.kubeclient.core_client }
subject { create(factory, cluster: cluster) }
before do
subject.cluster.platform_kubernetes.namespace = 'a-namespace'
stub_kubeclient_discover(cluster.platform_kubernetes.api_url)
create(:cluster_kubernetes_namespace,
cluster: cluster,
cluster_project: cluster.cluster_project,
project: cluster.cluster_project.project)
end
it 'creates proxy elasticsearch_client' do
expect(subject.elasticsearch_client).to be_instance_of(Elasticsearch::Transport::Client)
end
it 'copies proxy_url, options and headers from kube client to elasticsearch_client' do
expect(Elasticsearch::Client)
.to(receive(:new))
.with(url: a_valid_url)
.and_call_original
client = subject.elasticsearch_client
faraday_connection = client.transport.connections.first.connection
expect(faraday_connection.headers["Authorization"]).to eq(kube_client.headers[:Authorization])
expect(faraday_connection.ssl.cert_store).to be_instance_of(OpenSSL::X509::Store)
expect(faraday_connection.ssl.verify).to eq(1)
expect(faraday_connection.options.timeout).to be_nil
end
context 'when cluster is not reachable' do
before do
allow(kube_client).to receive(:proxy_url).and_raise(Kubeclient::HttpError.new(401, 'Unauthorized', nil))
end
it 'returns nil' do
expect(subject.elasticsearch_client).to be_nil
end
end
context 'when timeout is provided' do
it 'sets timeout in elasticsearch_client' do
client = subject.elasticsearch_client(timeout: 123)
faraday_connection = client.transport.connections.first.connection
expect(faraday_connection.options.timeout).to eq(123)
end
end
end
end
end
......@@ -2,7 +2,7 @@
RSpec.shared_examples '#create_or_update action' do
let(:params) do
{ integration: { application_type: Clusters::Applications::Prometheus.application_name, enabled: true } }
{ integration: { application_type: 'prometheus', enabled: true } }
end
let(:path) { raise NotImplementedError }
......
......@@ -11,6 +11,14 @@ elasticsearch:
filebeat:
enabled: true
extraVolumes:
- name: varlog
hostPath:
path: /var/log
extraVolumeMounts:
- name: varlog
mountPath: /var/log
readOnly: true
filebeatConfig:
filebeat.yml: |
output.file.enabled: false
......@@ -22,6 +30,28 @@ filebeat:
index: "filebeat-%{[agent.version]}-%{+yyyy.MM.dd}"
filebeat.inputs:
- type: container
format: cri
paths:
- '/var/log/containers/*.log'
json.keys_under_root: true
json.ignore_decoding_error: true
processors:
- add_id:
target_field: tie_breaker_id
- add_cloud_metadata: ~
- add_kubernetes_metadata:
host: ${NODE_NAME}
matchers:
- logs_path:
logs_path: "/var/log/containers/"
- decode_json_fields:
fields: ["message"]
when:
equals:
kubernetes.container.namespace: "gitlab-managed-apps"
kubernetes.container.name: "modsecurity-log"
- type: container
format: docker
paths:
- '/var/lib/docker/containers/*/*.log'
json.keys_under_root: true
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment