Commit f7d08041 authored by Jan Provaznik's avatar Jan Provaznik

Merge branch 'rc/add_custom_dashboard_importer' into 'master'

Add custom metric importer

See merge request gitlab-org/gitlab!37773
parents d2940636 2c05fd28
......@@ -20,7 +20,8 @@ module Enums
{
business: 0,
response: 1,
system: 2
system: 2,
custom: 3
}.freeze
end
......@@ -79,7 +80,11 @@ module Enums
system: {
group_title: _('System metrics (Custom)'),
priority: -10
}.freeze
}.freeze,
custom: {
group_title: _('Custom metrics'),
priority: 0
}
}.freeze
end
end
......
......@@ -16,11 +16,13 @@ class PrometheusMetric < ApplicationRecord
validates :project, presence: true, unless: :common?
validates :project, absence: true, if: :common?
scope :for_dashboard_path, -> (dashboard_path) { where(dashboard_path: dashboard_path) }
scope :for_project, -> (project) { where(project: project) }
scope :for_group, -> (group) { where(group: group) }
scope :for_title, -> (title) { where(title: title) }
scope :for_y_label, -> (y_label) { where(y_label: y_label) }
scope :for_identifier, -> (identifier) { where(identifier: identifier) }
scope :not_identifier, -> (identifier) { where.not(identifier: identifier) }
scope :common, -> { where(common: true) }
scope :ordered, -> { reorder(created_at: :asc) }
......
......@@ -18,6 +18,7 @@ module Gitlab
business: 0,
response: 1,
system: 2,
custom: 3,
cluster_health: -100
}
......@@ -34,7 +35,8 @@ module Gitlab
aws_elb: _('Response metrics (AWS ELB)'),
nginx: _('Response metrics (NGINX)'),
kubernetes: _('System metrics (Kubernetes)'),
cluster_health: _('Cluster Health')
cluster_health: _('Cluster Health'),
custom: _('Custom metrics')
}
end
end
......
# frozen_string_literal: true
module Gitlab
module Metrics
module Dashboard
class Importer
def initialize(dashboard_path, project)
@dashboard_path = dashboard_path.to_s
@project = project
end
def execute
return false unless Dashboard::Validator.validate(dashboard_hash, project: project, dashboard_path: dashboard_path)
Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute
rescue Gitlab::Config::Loader::FormatError
false
end
def execute!
Dashboard::Validator.validate!(dashboard_hash, project: project, dashboard_path: dashboard_path)
Dashboard::Importers::PrometheusMetrics.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute!
end
private
attr_accessor :dashboard_path, :project
def dashboard_hash
@dashboard_hash ||= begin
raw_dashboard = Dashboard::RepoDashboardFinder.read_dashboard(project, dashboard_path)
return unless raw_dashboard.present?
::Gitlab::Config::Loader::Yaml.new(raw_dashboard).load_raw!
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Metrics
module Dashboard
module Importers
class PrometheusMetrics
ALLOWED_ATTRIBUTES = %i(title query y_label unit legend group dashboard_path).freeze
# Takes a JSON schema validated dashboard hash and
# imports metrics to database
def initialize(dashboard_hash, project:, dashboard_path:)
@dashboard_hash = dashboard_hash
@project = project
@dashboard_path = dashboard_path
end
def execute
import
rescue ActiveRecord::RecordInvalid, ::Gitlab::Metrics::Dashboard::Transformers::TransformerError
false
end
def execute!
import
end
private
attr_reader :dashboard_hash, :project, :dashboard_path
def import
delete_stale_metrics
create_or_update_metrics
end
# rubocop: disable CodeReuse/ActiveRecord
def create_or_update_metrics
# TODO: use upsert and worker for callbacks?
prometheus_metrics_attributes.each do |attributes|
prometheus_metric = PrometheusMetric.find_or_initialize_by(attributes.slice(:identifier, :project))
prometheus_metric.update!(attributes.slice(*ALLOWED_ATTRIBUTES))
end
end
# rubocop: enable CodeReuse/ActiveRecord
def delete_stale_metrics
identifiers = prometheus_metrics_attributes.map { |metric_attributes| metric_attributes[:identifier] }
stale_metrics = PrometheusMetric.for_project(project)
.for_dashboard_path(dashboard_path)
.for_group(Enums::PrometheusMetric.groups[:custom])
.not_identifier(identifiers)
# TODO: use destroy_all and worker for callbacks?
stale_metrics.each(&:destroy)
end
def prometheus_metrics_attributes
@prometheus_metrics_attributes ||= begin
Dashboard::Transformers::Yml::V1::PrometheusMetrics.new(
dashboard_hash,
project: project,
dashboard_path: dashboard_path
).execute
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Metrics
module Dashboard
module Transformers
TransformerError = Class.new(StandardError)
module Errors
class MissingAttribute < TransformerError
def initialize(attribute_name)
super("Missing attribute: '#{attribute_name}'")
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Metrics
module Dashboard
module Transformers
module Yml
module V1
# Takes a JSON schema validated dashboard hash and
# maps it to PrometheusMetric model attributes
class PrometheusMetrics
def initialize(dashboard_hash, project: nil, dashboard_path: nil)
@dashboard_hash = dashboard_hash.with_indifferent_access
@project = project
@dashboard_path = dashboard_path
@dashboard_hash.default_proc = -> (h, k) { raise Transformers::Errors::MissingAttribute, k.to_s }
end
def execute
prometheus_metrics = []
dashboard_hash[:panel_groups].each do |panel_group|
panel_group[:panels].each do |panel|
panel[:metrics].each do |metric|
prometheus_metrics << {
project: project,
title: panel[:title],
y_label: panel[:y_label],
query: metric[:query_range] || metric[:query],
unit: metric[:unit],
legend: metric[:label],
identifier: metric[:id],
group: Enums::PrometheusMetric.groups[:custom],
common: false,
dashboard_path: dashboard_path
}.compact
end
end
end
prometheus_metrics
end
private
attr_reader :dashboard_hash, :project, :dashboard_path
end
end
end
end
end
end
end
......@@ -7426,6 +7426,9 @@ msgstr ""
msgid "Custom hostname (for private commit emails)"
msgstr ""
msgid "Custom metrics"
msgstr ""
msgid "Custom notification events"
msgstr ""
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Metrics::Dashboard::Importer do
include MetricsDashboardHelpers
let_it_be(:dashboard_path) { '.gitlab/dashboards/sample_dashboard.yml' }
let_it_be(:project) { create(:project) }
before do
allow(subject).to receive(:dashboard_hash).and_return(dashboard_hash)
end
subject { described_class.new(dashboard_path, project) }
describe '.execute' do
context 'valid dashboard hash' do
let(:dashboard_hash) { load_sample_dashboard }
it 'imports metrics to database' do
expect { subject.execute }
.to change { PrometheusMetric.count }.from(0).to(3)
end
end
context 'invalid dashboard hash' do
let(:dashboard_hash) { {} }
it 'returns false' do
expect(subject.execute).to be(false)
end
end
end
describe '.execute!' do
context 'valid dashboard hash' do
let(:dashboard_hash) { load_sample_dashboard }
it 'imports metrics to database' do
expect { subject.execute }
.to change { PrometheusMetric.count }.from(0).to(3)
end
end
context 'invalid dashboard hash' do
let(:dashboard_hash) { {} }
it 'raises error' do
expect { subject.execute! }.to raise_error(Gitlab::Metrics::Dashboard::Validator::Errors::SchemaValidationError,
'root is missing required keys: dashboard, panel_groups')
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Metrics::Dashboard::Importers::PrometheusMetrics do
include MetricsDashboardHelpers
describe '#execute' do
let(:project) { create(:project) }
let(:dashboard_path) { 'path/to/dashboard.yml' }
subject { described_class.new(dashboard_hash, project: project, dashboard_path: dashboard_path) }
context 'valid dashboard' do
let(:dashboard_hash) { load_sample_dashboard }
context 'with all new metrics' do
it 'creates PrometheusMetrics' do
expect { subject.execute }.to change { PrometheusMetric.count }.by(3)
end
end
context 'with existing metrics' do
let!(:existing_metric) do
create(:prometheus_metric, {
project: project,
identifier: 'metric_b',
title: 'overwrite',
y_label: 'overwrite',
query: 'overwrite',
unit: 'overwrite',
legend: 'overwrite'
})
end
it 'updates existing PrometheusMetrics' do
described_class.new(dashboard_hash, project: project, dashboard_path: dashboard_path).execute
expect(existing_metric.reload.attributes.with_indifferent_access).to include({
title: 'Super Chart B',
y_label: 'y_label',
query: 'query',
unit: 'unit',
legend: 'Legend Label'
})
end
it 'creates new PrometheusMetrics' do
expect { subject.execute }.to change { PrometheusMetric.count }.by(2)
end
context 'with stale metrics' do
let!(:stale_metric) do
create(:prometheus_metric,
project: project,
identifier: 'stale_metric',
dashboard_path: dashboard_path,
group: 3
)
end
it 'deletes stale metrics' do
subject.execute
expect { stale_metric.reload }.to raise_error(ActiveRecord::RecordNotFound)
end
end
end
end
context 'invalid dashboard' do
let(:dashboard_hash) { {} }
it 'returns false' do
expect(subject.execute).to eq(false)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Metrics::Dashboard::Transformers::Yml::V1::PrometheusMetrics do
include MetricsDashboardHelpers
describe '#execute' do
subject { described_class.new(dashboard_hash) }
context 'valid dashboard' do
let_it_be(:dashboard_hash) do
{
panel_groups: [{
panels: [
{
title: 'Panel 1 title',
y_label: 'Panel 1 y_label',
metrics: [
{
query_range: 'Panel 1 metric 1 query_range',
unit: 'Panel 1 metric 1 unit',
label: 'Panel 1 metric 1 label',
id: 'Panel 1 metric 1 id'
},
{
query: 'Panel 1 metric 2 query',
unit: 'Panel 1 metric 2 unit',
label: 'Panel 1 metric 2 label',
id: 'Panel 1 metric 2 id'
}
]
},
{
title: 'Panel 2 title',
y_label: 'Panel 2 y_label',
metrics: [{
query_range: 'Panel 2 metric 1 query_range',
unit: 'Panel 2 metric 1 unit',
label: 'Panel 2 metric 1 label',
id: 'Panel 2 metric 1 id'
}]
}
]
}]
}
end
let(:expected_metrics) do
[
{
title: 'Panel 1 title',
y_label: 'Panel 1 y_label',
query: "Panel 1 metric 1 query_range",
unit: 'Panel 1 metric 1 unit',
legend: 'Panel 1 metric 1 label',
identifier: 'Panel 1 metric 1 id',
group: 3,
common: false
},
{
title: 'Panel 1 title',
y_label: 'Panel 1 y_label',
query: 'Panel 1 metric 2 query',
unit: 'Panel 1 metric 2 unit',
legend: 'Panel 1 metric 2 label',
identifier: 'Panel 1 metric 2 id',
group: 3,
common: false
},
{
title: 'Panel 2 title',
y_label: 'Panel 2 y_label',
query: 'Panel 2 metric 1 query_range',
unit: 'Panel 2 metric 1 unit',
legend: 'Panel 2 metric 1 label',
identifier: 'Panel 2 metric 1 id',
group: 3,
common: false
}
]
end
it 'returns collection of metrics with correct attributes' do
expect(subject.execute).to match_array(expected_metrics)
end
end
context 'invalid dashboard' do
let(:dashboard_hash) { {} }
it 'raises missing attribute error' do
expect { subject.execute }.to raise_error(
::Gitlab::Metrics::Dashboard::Transformers::Errors::MissingAttribute, "Missing attribute: 'panel_groups'"
)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment