Commit cb949ee0 authored by charlie ablett's avatar charlie ablett

Merge branch '208224-move-cluster-health-feature-into-gitlab-core' into 'master'

[RUN AS-IF-FOSS] Move cluster health metrics dashboard serivces into gitlab core

See merge request gitlab-org/gitlab!35873
parents faca8aaf 8a19a256
......@@ -10,29 +10,29 @@ module Metrics
ALLOWED_FILE_TYPE = '.yml'
USER_DASHBOARDS_DIR = ::Metrics::Dashboard::CustomDashboardService::DASHBOARD_ROOT
SEQUENCES = {
::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
].freeze,
::Metrics::Dashboard::SelfMonitoringDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter
].freeze,
::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
].freeze
}.freeze
steps :check_push_authorized,
:check_branch_name,
:check_file_type,
:check_dashboard_template,
:create_file,
:refresh_repository_method_caches
class << self
def sequences
@sequences ||= {
::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
].freeze,
::Metrics::Dashboard::SelfMonitoringDashboardService::DASHBOARD_PATH => [
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter
].freeze
}.freeze
end
end
:check_branch_name,
:check_file_type,
:check_dashboard_template,
:create_file,
:refresh_repository_method_caches
def execute
execute_steps
......@@ -173,10 +173,8 @@ module Metrics
end
def sequence
self.class.sequences[dashboard_template] || []
SEQUENCES[dashboard_template] || []
end
end
end
end
Metrics::Dashboard::CloneDashboardService.prepend_if_ee('EE::Metrics::Dashboard::CloneDashboardService')
......@@ -5,7 +5,7 @@
module Metrics
module Dashboard
class ClusterDashboardService < ::Metrics::Dashboard::PredefinedDashboardService
DASHBOARD_PATH = 'ee/config/prometheus/cluster_metrics.yml'
DASHBOARD_PATH = 'config/prometheus/cluster_metrics.yml'
DASHBOARD_NAME = 'Cluster'
SEQUENCE = [
......
dashboard: 'Cluster health'
priority: 1
panel_groups:
- group: Cluster Health
priority: 1
metrics:
priority: 10
panels:
- title: "CPU Usage"
type: "area-chart"
y_label: "CPU (cores)"
required_metrics: ['container_cpu_usage_seconds_total']
weight: 1
queries:
- query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)'
label: Usage (cores)
unit: "cores"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
label: Requested (cores)
unit: "cores"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
label: Capacity (cores)
unit: "cores"
appearance:
line:
type: 'dashed'
width: 2
area:
opacity: 0
- title: "Memory usage"
metrics:
- id: cluster_health_cpu_usage
query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)'
unit: cores
label: Usage (cores)
- id: cluster_health_cpu_requested
query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: cores
label: Requested (cores)
- id: cluster_health_cpu_capacity
query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: cores
label: Capacity (cores)
- title: "Memory Usage"
type: "area-chart"
y_label: "Memory (GiB)"
required_metrics: ['container_memory_usage_bytes']
weight: 1
queries:
- query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30'
label: Usage (GiB)
unit: "GiB"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
label: Requested (GiB)
unit: "GiB"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
label: Capacity (GiB)
unit: "GiB"
appearance:
line:
type: 'dashed'
width: 2
area:
opacity: 0
metrics:
- id: cluster_health_memory_usage
query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30'
unit: GiB
label: Usage (GiB)
- id: cluster_health_memory_requested
query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: GiB
label: Requested (GiB)
- id: cluster_health_memory_capacity
query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: GiB
label: Capacity (GiB)
# most likely this file can be removed, but until we are sure and have capacity to tackle that I've
# only moved it and added https://gitlab.com/gitlab-org/gitlab/-/issues/225869 to track work need to clean up codebase.
- group: Cluster Health
priority: 1
metrics:
- title: "CPU Usage"
y_label: "CPU (cores)"
required_metrics: ['container_cpu_usage_seconds_total']
weight: 1
queries:
- query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)'
label: Usage (cores)
unit: "cores"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
label: Requested (cores)
unit: "cores"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
label: Capacity (cores)
unit: "cores"
appearance:
line:
type: 'dashed'
width: 2
area:
opacity: 0
- title: "Memory usage"
y_label: "Memory (GiB)"
required_metrics: ['container_memory_usage_bytes']
weight: 1
queries:
- query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30'
label: Usage (GiB)
unit: "GiB"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
label: Requested (GiB)
unit: "GiB"
appearance:
line:
width: 2
area:
opacity: 0
- query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
label: Capacity (GiB)
unit: "GiB"
appearance:
line:
type: 'dashed'
width: 2
area:
opacity: 0
# frozen_string_literal: true
# Copies system dashboard definition in .yml file into designated
# .yml file inside `.gitlab/dashboards`
module EE
module Metrics
module Dashboard
module CloneDashboardService
extend ActiveSupport::Concern
class_methods do
extend ::Gitlab::Utils::Override
override :sequences
def sequences
super.merge(::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH => [::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter].freeze)
end
end
end
end
end
end
dashboard: 'Cluster health'
priority: 1
panel_groups:
- group: Cluster Health
priority: 10
panels:
- title: "CPU Usage"
type: "area-chart"
y_label: "CPU (cores)"
weight: 1
metrics:
- id: cluster_health_cpu_usage
query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)'
unit: cores
label: Usage (cores)
- id: cluster_health_cpu_requested
query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: cores
label: Requested (cores)
- id: cluster_health_cpu_capacity
query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: cores
label: Capacity (cores)
- title: "Memory Usage"
type: "area-chart"
y_label: "Memory (GiB)"
weight: 1
metrics:
- id: cluster_health_memory_usage
query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30'
unit: GiB
label: Usage (GiB)
- id: cluster_health_memory_requested
query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: GiB
label: Requested (GiB)
- id: cluster_health_memory_capacity
query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: GiB
label: Capacity (GiB)
# frozen_string_literal: true
module EE
module Gitlab
module Metrics
module Dashboard
module ServiceSelector
extend ActiveSupport::Concern
EE_SERVICES = [
::Metrics::Dashboard::ClusterMetricsEmbedService,
::Metrics::Dashboard::ClusterDashboardService
].freeze
class_methods do
extend ::Gitlab::Utils::Override
private
override :services
def services
EE_SERVICES + super
end
end
end
end
end
end
end
......@@ -6,22 +6,6 @@ module EE
module Metrics
module Dashboard
module Url
# Matches dashboard urls for a metric chart embed
# for cluster metrics
#
# EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
def clusters_regex
strong_memoize(:clusters_regex) do
regex_for_project_metrics(
%r{
/clusters
/(?<cluster_id>\d+)
/?
}x
)
end
end
# Matches dashboard urls for a metric chart embed
# for a specifc firing GitLab alert
#
......
......@@ -7,7 +7,7 @@ module Gitlab
include QueryAdditionalMetrics
def query
AdditionalMetricsParser.load_groups_from_yaml('cluster_metrics.yml')
AdditionalMetricsParser.load_groups_from_yaml('queries_cluster_metrics.yml')
.map(&query_group(base_query_context(8.hours.ago, Time.now)))
end
end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Metrics::Dashboard::ServiceSelector do
include MetricsDashboardHelpers
describe '#call' do
subject { described_class.call(arguments) }
context 'when cluster is provided' do
let(:arguments) { { cluster: "some cluster" } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster is provided and embedded is not true' do
let(:arguments) { { cluster: "some cluster", embedded: 'false' } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster dashboard_path is provided' do
let(:arguments) { { dashboard_path: ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster is provided and embed params' do
let(:arguments) do
{
cluster: "some cluster",
embedded: 'true',
cluster_type: 'project',
format: :json,
group: 'Food metrics',
title: 'Pizza Consumption',
y_label: 'Slice Count'
}
end
it { is_expected.to be Metrics::Dashboard::ClusterMetricsEmbedService }
end
context 'when metrics embed is for an alert' do
let(:arguments) { { embedded: true, prometheus_alert_id: 5 } }
it { is_expected.to be Metrics::Dashboard::GitlabAlertEmbedService }
end
end
end
......@@ -3,35 +3,6 @@
require 'spec_helper'
RSpec.describe Gitlab::Metrics::Dashboard::Url do
describe '#clusters_regex' do
let(:url) do
Gitlab::Routing.url_helpers.namespace_project_cluster_url(
'foo',
'bar',
'1',
group: 'Cluster Health',
title: 'Memory Usage',
y_label: 'Memory 20(GiB)',
anchor: 'title'
)
end
let(:expected_params) do
{
'url' => url,
'namespace' => 'foo',
'project' => 'bar',
'cluster_id' => '1',
'query' => '?group=Cluster+Health&title=Memory+Usage&y_label=Memory+20%28GiB%29',
'anchor' => '#title'
}
end
subject { described_class.clusters_regex }
it_behaves_like 'regex which matches url when expected'
end
describe '#alert_regex' do
let(:url) do
Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_prometheus_alert_url(
......
......@@ -12,7 +12,7 @@ RSpec.describe Gitlab::Prometheus::Queries::ClusterQuery do
end
it 'load cluster metrics from yaml' do
expect(Gitlab::Prometheus::AdditionalMetricsParser).to receive(:load_groups_from_yaml).with('cluster_metrics.yml').and_call_original
expect(Gitlab::Prometheus::AdditionalMetricsParser).to receive(:load_groups_from_yaml).with('queries_cluster_metrics.yml').and_call_original
subject.query
end
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Metrics::Dashboard::CloneDashboardService, :use_clean_rails_memory_store_caching do
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :repository) }
let_it_be(:environment) { create(:environment, project: project) }
describe '#execute' do
subject(:service_call) { described_class.new(project, user, params).execute }
context 'with rights to push to the repository' do
before do
project.add_maintainer(user)
end
context 'valid parameters' do
let(:commit_message) { 'test' }
let(:branch) { "#{Time.current.to_i}_dashboard_new_branch" }
let(:file_name) { 'custom_dashboard.yml' }
let(:file_content_hash) { YAML.safe_load(File.read(dashboard)) }
let(:params) do
{
dashboard: dashboard,
file_name: file_name,
commit_message: commit_message,
branch: branch
}
end
let(:stages) { ::Gitlab::Metrics::Dashboard::Stages }
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH,
[
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
]
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH,
[
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
]
end
end
end
end
......@@ -13,6 +13,8 @@ module Gitlab
include Gitlab::Utils::StrongMemoize
SERVICES = [
::Metrics::Dashboard::ClusterMetricsEmbedService,
::Metrics::Dashboard::ClusterDashboardService,
::Metrics::Dashboard::GitlabAlertEmbedService,
::Metrics::Dashboard::CustomMetricEmbedService,
::Metrics::Dashboard::GrafanaMetricEmbedService,
......@@ -51,5 +53,3 @@ module Gitlab
end
end
end
Gitlab::Metrics::Dashboard::ServiceSelector.prepend_if_ee('EE::Gitlab::Metrics::Dashboard::ServiceSelector')
......@@ -60,6 +60,22 @@ module Gitlab
Gitlab::Routing.url_helpers.metrics_dashboard_namespace_project_environment_url(*args)
end
# Matches dashboard urls for a metric chart embed
# for cluster metrics
#
# EX - https://<host>/<namespace>/<project>/-/clusters/<cluster_id>/?group=Cluster%20Health&title=Memory%20Usage&y_label=Memory%20(GiB)
def clusters_regex
strong_memoize(:clusters_regex) do
regex_for_project_metrics(
%r{
/clusters
/(?<cluster_id>\d+)
/?
}x
)
end
end
private
def regex_for_project_metrics(path_suffix_pattern)
......
......@@ -109,6 +109,46 @@ RSpec.describe Gitlab::Metrics::Dashboard::ServiceSelector do
it { is_expected.to be Metrics::Dashboard::TransientEmbedService }
end
context 'when cluster is provided' do
let(:arguments) { { cluster: "some cluster" } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster is provided and embedded is not true' do
let(:arguments) { { cluster: "some cluster", embedded: 'false' } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster dashboard_path is provided' do
let(:arguments) { { dashboard_path: ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
context 'when cluster is provided and embed params' do
let(:arguments) do
{
cluster: "some cluster",
embedded: 'true',
cluster_type: 'project',
format: :json,
group: 'Food metrics',
title: 'Pizza Consumption',
y_label: 'Slice Count'
}
end
it { is_expected.to be Metrics::Dashboard::ClusterMetricsEmbedService }
end
context 'when metrics embed is for an alert' do
let(:arguments) { { embedded: true, prometheus_alert_id: 5 } }
it { is_expected.to be Metrics::Dashboard::GitlabAlertEmbedService }
end
end
end
end
......@@ -46,6 +46,35 @@ RSpec.describe Gitlab::Metrics::Dashboard::Url do
end
end
describe '#clusters_regex' do
let(:url) do
Gitlab::Routing.url_helpers.namespace_project_cluster_url(
'foo',
'bar',
'1',
group: 'Cluster Health',
title: 'Memory Usage',
y_label: 'Memory 20(GiB)',
anchor: 'title'
)
end
let(:expected_params) do
{
'url' => url,
'namespace' => 'foo',
'project' => 'bar',
'cluster_id' => '1',
'query' => '?group=Cluster+Health&title=Memory+Usage&y_label=Memory+20%28GiB%29',
'anchor' => '#title'
}
end
subject { described_class.clusters_regex }
it_behaves_like 'regex which matches url when expected'
end
describe '#grafana_regex' do
let(:url) do
namespace_project_grafana_api_metrics_dashboard_url(
......
......@@ -81,7 +81,18 @@ RSpec.describe Metrics::Dashboard::CloneDashboardService, :use_clean_rails_memor
allow(::Gitlab::Metrics::Dashboard::Processor).to receive(:new).and_return(double(process: file_content_hash))
end
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH, [::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter, ::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter, ::Gitlab::Metrics::Dashboard::Stages::Sorter]
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::SystemDashboardService::DASHBOARD_PATH,
[
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::CustomMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
]
it_behaves_like 'valid dashboard cloning process', ::Metrics::Dashboard::ClusterDashboardService::DASHBOARD_PATH,
[
::Gitlab::Metrics::Dashboard::Stages::CommonMetricsInserter,
::Gitlab::Metrics::Dashboard::Stages::Sorter
]
it_behaves_like 'valid dashboard cloning process',
::Metrics::Dashboard::SelfMonitoringDashboardService::DASHBOARD_PATH,
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment