Commit 5839ec18 authored by Ryan Cobb's avatar Ryan Cobb

Add cluster metrics dashboard processing

This adds cluster metrics dashboard processing. This will be used
in addition with cluster prometheus proxies to move cluster dashboard
metrics to a similar pattern that we use for environment metrics.
parent 4615c56d
......@@ -11,3 +11,5 @@ class Admin::ClustersController < Clusters::ClustersController
@clusterable ||= InstanceClusterablePresenter.fabricate(Clusters::Instance.new, current_user: current_user)
end
end
Admin::ClustersController.prepend_if_ee('EE::Admin::ClustersController')
......@@ -24,3 +24,5 @@ class Projects::ClustersController < Clusters::ClustersController
@repository ||= project.repository
end
end
Projects::ClustersController.prepend_if_ee('EE::Projects::ClustersController')
......@@ -31,14 +31,24 @@ module Metrics
# Determines whether users should be able to view
# dashboards at all.
def allowed?
Ability.allowed?(current_user, :read_environment, project)
if params[:cluster]
true # Authorization handled at controller level
else
Ability.allowed?(current_user, :read_environment, project)
end
end
# Returns a new dashboard Hash, supplemented with DB info
def process_dashboard
Gitlab::Metrics::Dashboard::Processor
.new(project, params[:environment], raw_dashboard)
.process(insert_project_metrics: insert_project_metrics?)
if params[:cluster]
::Gitlab::Metrics::Dashboard::ClusterProcessor
.new(project, raw_dashboard, params)
.process
else
::Gitlab::Metrics::Dashboard::Processor
.new(project, raw_dashboard, params)
.process(insert_project_metrics: insert_project_metrics?)
end
end
# @return [String] Relative filepath of the dashboard yml
......
......@@ -143,7 +143,8 @@ Rails.application.routes.draw do
member do
Gitlab.ee do
get :metrics, format: :json
get '/prometheus/api/v1/*proxy_path', to: 'clusters#prometheus_proxy', as: :prometheus_api
get :metrics_dashboard
get :'/prometheus/api/v1/*proxy_path', to: 'clusters#prometheus_proxy', as: :prometheus_api
end
scope :applications do
......
# frozen_string_literal: true
module EE
module Admin
module ClustersController
def metrics_dashboard_params
{
cluster: cluster,
cluster_type: :admin
}
end
end
end
end
......@@ -49,9 +49,33 @@ module EE
)
end
end
def metrics_dashboard
project_for_dashboard = defined?(project) ? project : nil
dashboard = dashboard_finder.find(project_for_dashboard, current_user, nil, metrics_dashboard_params)
respond_to do |format|
if dashboard[:status] == :success
format.json do
render status: :ok, json: dashboard.slice(:all_dashboards, :dashboard, :status)
end
else
format.json do
render(
status: dashboard[:http_status],
json: dashboard.slice(:all_dashboards, :message, :status)
)
end
end
end
end
private
def dashboard_finder
::Gitlab::Metrics::Dashboard::Finder
end
def prometheus_adapter
return unless cluster&.application_prometheus_available?
......
......@@ -25,6 +25,14 @@ module EE
.with_pagination(request, response)
.represent(environments)
end
def metrics_dashboard_params
{
cluster: cluster,
cluster_type: :group,
group: group
}
end
end
end
end
# frozen_string_literal: true
module EE
module Projects
module ClustersController
def metrics_dashboard_params
{
cluster: cluster,
cluster_type: :project
}
end
end
end
end
......@@ -24,6 +24,10 @@ module EE
true
end
def prometheus_adapter
application_prometheus
end
end
end
end
# frozen_string_literal: true
# Fetches the system metrics dashboard and formats the output.
# Use Gitlab::Metrics::Dashboard::Finder to retrive dashboards.
module Metrics
module Dashboard
class ClusterDashboardService < ::Metrics::Dashboard::BaseService
CLUSTER_DASHBOARD_PATH = 'ee/config/prometheus/cluster_metrics_new.yml'
CLUSTER_DASHBOARD_NAME = 'Cluster'
class << self
def all_dashboard_paths(_project)
[{
path: CLUSTER_DASHBOARD_PATH,
display_name: CLUSTER_DASHBOARD_NAME,
default: true
}]
end
end
private
def dashboard_path
CLUSTER_DASHBOARD_PATH
end
# Returns the base metrics shipped with every GitLab service.
def get_raw_dashboard
yml = File.read(Rails.root.join(dashboard_path))
YAML.safe_load(yml)
end
def cache_key
"metrics_dashboard_#{dashboard_path}"
end
def insert_project_metrics?
false
end
end
end
end
dashboard: 'Cluster health'
priority: 1
panel_groups:
- group: Cluster Health
priority: 10
panels:
- title: "CPU Usage"
type: "area-chart"
y_label: "CPU (cores)"
weight: 1
metrics:
- id: cluster_health_cpu_usage
query_range: 'avg(sum(rate(container_cpu_usage_seconds_total{id="/"}[15m])) by (job)) without (job)'
unit: cores
label: Usage (cores)
- id: cluster_health_cpu_requested
query_range: 'sum(kube_pod_container_resource_requests_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: cores
label: Requested (cores)
- id: cluster_health_cpu_capacity
query_range: 'sum(kube_node_status_capacity_cpu_cores{kubernetes_namespace="gitlab-managed-apps"})'
unit: cores
label: Capacity (cores)
- title: "Memory Usage"
type: "area-chart"
y_label: "Memory (GiB)"
weight: 1
metrics:
- id: cluster_health_memory_usage
query_range: 'avg(sum(container_memory_usage_bytes{id="/"}) by (job)) without (job) / 2^30'
unit: GiB
label: Usage (GiB)
- id: cluster_health_memory_requested
query_range: 'sum(kube_pod_container_resource_requests_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: GiB
label: Requested (GiB)
- id: cluster_health_memory_capacity
query_range: 'sum(kube_node_status_capacity_memory_bytes{kubernetes_namespace="gitlab-managed-apps"})/2^30'
unit: GiB
label: Capacity (GiB)
# frozen_string_literal: true
module EE
module Gitlab
module Metrics
module Dashboard
module ServiceSelector
extend ActiveSupport::Concern
class_methods do
def call(params)
return ::Metrics::Dashboard::ClusterDashboardService if params[:cluster]
super
end
end
end
end
end
end
end
......@@ -16,7 +16,7 @@ module EE
for_metrics do |metric|
next unless metrics_with_alerts.include?(metric[:metric_id])
metric[:alert_path] = alert_path(metric[:metric_id], project, environment)
metric[:alert_path] = alert_path(metric[:metric_id], project, params[:environment])
end
end
......@@ -25,7 +25,7 @@ module EE
def metrics_with_alerts
strong_memoize(:metrics_with_alerts) do
alerts = ::Projects::Prometheus::AlertsFinder
.new(project: project, environment: environment)
.new(project: project, environment: params[:environment])
.execute
Set.new(alerts.map(&:prometheus_metric_id))
......
# frozen_string_literal: true
module Gitlab
module Metrics
module Dashboard
# Responsible for processesing a dashboard hash, inserting
# relevant DB records & sorting for proper rendering in
# the UI. These includes shared metric info, custom metrics
# info, and alerts (only in EE).
class ClusterProcessor
SEQUENCE = [
Stages::CommonMetricsInserter,
Stages::ClusterEndpointInserter,
Stages::Sorter
].freeze
def initialize(project, dashboard, params)
@project = project
@dashboard = dashboard
@params = params
end
# Returns a new dashboard hash with the results of
# running transforms on the dashboard.
def process
@dashboard.deep_symbolize_keys.tap do |dashboard|
SEQUENCE.each do |stage|
stage.new(@project, dashboard, @params).transform!
end
end
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module Metrics
module Dashboard
module Stages
class ClusterEndpointInserter < BaseStage
def transform!
verify_params
for_metrics do |metric|
metric[:prometheus_endpoint_path] = endpoint_for_metric(metric)
end
end
private
def endpoint_for_metric(metric)
proxy_path = query_type(metric)
query = query_for_metric(metric)
case params[:cluster_type]
when :admin
Gitlab::Routing.url_helpers.prometheus_api_admin_cluster_path(
params[:cluster],
proxy_path: proxy_path,
query: query
)
when :group
Gitlab::Routing.url_helpers.prometheus_api_group_cluster_path(
params[:group],
params[:cluster],
proxy_path: proxy_path,
query: query
)
when :project
Gitlab::Routing.url_helpers.prometheus_api_project_cluster_path(
project,
params[:cluster],
proxy_path: proxy_path,
query: query
)
else
Errors::DashboardProcessingError.new('Unrecognized cluster type')
end
end
def query_type(metric)
metric[:query] ? :query : :query_range
end
def query_for_metric(metric)
query = metric[query_type(metric)]
raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query
query
end
def verify_params
raise Errors::DashboardProcessingError.new('Cluster is required for Stages::ClusterEndpointInserter') unless params[:cluster]
raise Errors::DashboardProcessingError.new('Cluster type must be specificed for Stages::ClusterEndpointInserter') unless params[:cluster_type]
verify_type_params
end
def verify_type_params
case params[:cluster_type]
when :group
raise Errors::DashboardProcessingError.new('Group is required when cluster_type is :group') unless params[:group]
when :project
raise Errors::DashboardProcessingError.new('Project is required when cluster_type is :project') unless project
end
end
end
end
end
end
end
......@@ -6,7 +6,7 @@ describe Gitlab::Metrics::Dashboard::Processor do
let(:project) { build(:project) }
let(:environment) { create(:environment, project: project) }
let(:dashboard_yml) { YAML.load_file('spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
let(:params) { [project, environment, dashboard_yml] }
let(:params) { [project, dashboard_yml, { environment: environment }] }
describe 'sequence' do
let(:environment) { build(:environment) }
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Metrics::Dashboard::ServiceSelector do
include MetricsDashboardHelpers
describe '#call' do
subject { described_class.call(arguments) }
context 'when cluster is provided' do
let(:arguments) { { cluster: "some cluster" } }
it { is_expected.to be Metrics::Dashboard::ClusterDashboardService }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Metrics::Dashboard::ClusterProcessor do
let(:cluster_project) { build(:cluster_project) }
let(:cluster) { cluster_project.cluster }
let(:project) { cluster_project.project }
let(:dashboard_yml) { YAML.load_file('spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
describe 'process' do
let(:process_params) { [project, dashboard_yml, { cluster: cluster, cluster_type: :project }] }
let(:dashboard) { described_class.new(*process_params).process }
it 'includes a path for the prometheus endpoint with each metric' do
expect(all_metrics).to satisfy_all do |metric|
metric[:prometheus_endpoint_path] == prometheus_path(metric[:query_range])
end
end
context 'when dashboard config corresponds to common metrics' do
let!(:common_metric) { create(:prometheus_metric, :common, identifier: 'metric_a1') }
it 'inserts metric ids into the config' do
target_metric = all_metrics.find { |metric| metric[:id] == 'metric_a1' }
expect(target_metric).to include(:metric_id)
expect(target_metric[:metric_id]).to eq(common_metric.id)
end
end
shared_examples_for 'errors with message' do |expected_message|
it 'raises a DashboardLayoutError' do
error_class = Gitlab::Metrics::Dashboard::Errors::DashboardProcessingError
expect { dashboard }.to raise_error(error_class, expected_message)
end
end
context 'when the dashboard is missing panel_groups' do
let(:dashboard_yml) { {} }
it_behaves_like 'errors with message', 'Top-level key :panel_groups must be an array'
end
context 'when the dashboard contains a panel_group which is missing panels' do
let(:dashboard_yml) { { panel_groups: [{}] } }
it_behaves_like 'errors with message', 'Each "panel_group" must define an array :panels'
end
context 'when the dashboard contains a panel which is missing metrics' do
let(:dashboard_yml) { { panel_groups: [{ panels: [{}] }] } }
it_behaves_like 'errors with message', 'Each "panel" must define an array :metrics'
end
context 'when the dashboard contains a metric which is missing a query' do
let(:dashboard_yml) { { panel_groups: [{ panels: [{ metrics: [{}] }] }] } }
it_behaves_like 'errors with message', 'Each "metric" must define one of :query or :query_range'
end
end
private
def all_metrics
dashboard[:panel_groups].flat_map do |group|
group[:panels].flat_map { |panel| panel[:metrics] }
end
end
def get_metric_details(metric)
{
query_range: metric.query,
unit: metric.unit,
label: metric.legend,
metric_id: metric.id,
prometheus_endpoint_path: prometheus_path(metric.query)
}
end
def prometheus_path(query)
Gitlab::Routing.url_helpers.prometheus_api_project_cluster_path(
project,
cluster,
proxy_path: :query_range,
query: query
)
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Metrics::Dashboard::ClusterDashboardService, :use_clean_rails_memory_store_caching do
include MetricsDashboardHelpers
set(:user) { create(:user) }
set(:cluster_project) { create(:cluster_project) }
set(:cluster) { cluster_project.cluster }
set(:project) { cluster_project.project }
before do
project.add_maintainer(user)
end
describe 'get_dashboard' do
let(:dashboard_path) { described_class::CLUSTER_DASHBOARD_PATH }
let(:service_params) { [project, user, { cluster: cluster, cluster_type: :project, dashboard_path: dashboard_path }] }
let(:service_call) { described_class.new(*service_params).get_dashboard }
it_behaves_like 'valid dashboard service response'
it 'caches the unprocessed dashboard for subsequent calls' do
expect(YAML).to receive(:safe_load).once.and_call_original
described_class.new(*service_params).get_dashboard
described_class.new(*service_params).get_dashboard
end
context 'when called with a non-system dashboard' do
let(:dashboard_path) { 'garbage/dashboard/path' }
# We want to alwaus return the system dashboard.
it_behaves_like 'valid dashboard service response'
end
end
describe '::all_dashboard_paths' do
it 'returns the dashboard attributes' do
all_dashboards = described_class.all_dashboard_paths(project)
expect(all_dashboards).to eq(
[{
path: described_class::CLUSTER_DASHBOARD_PATH,
display_name: described_class::CLUSTER_DASHBOARD_NAME,
default: true
}]
)
end
end
end
......@@ -186,6 +186,58 @@ shared_examples 'cluster metrics' do
end
end
end
describe 'GET #metrics_dashboard' do
let(:user) { create(:user) }
before do
clusterable.add_maintainer(user)
sign_in(user)
end
shared_examples_for 'correctly formatted response' do |status_code|
it 'returns a json object with the correct keys' do
get :metrics_dashboard, params: metrics_params, format: :json
found_keys = json_response.keys - ['all_dashboards']
expect(response).to have_gitlab_http_status(status_code)
expect(found_keys).to contain_exactly(*expected_keys)
end
end
shared_examples_for '200 response' do
let(:expected_keys) { %w(dashboard status) }
it_behaves_like 'correctly formatted response', :ok
end
shared_context 'error response' do |status_code|
let(:expected_keys) { %w(message status) }
it_behaves like 'correctly formatted response', status_code
end
shared_examples_for 'includes all dashboards' do
it 'includes info for all findable dashboards' do
expect(json_response).to have_key('all_dashboards')
expect(json_response['all_dashboards']).to be_an_instance_of(Array)
expect(json_response['all_dashboards']).to all( include('path', 'default', 'display_name') )
end
end
shared_examples_for 'the default dashboard' do
it_behaves_like '200 response'
it 'is the default dashboard' do
get :metrics_dashboard, params: metrics_params, format: :json
expect(json_response['dashboard']['dashboard']).to eq('Cluster health')
end
end
it_behaves_like 'the default dashboard'
end
end
private
......
......@@ -7,14 +7,16 @@ module Gitlab
module Metrics
module Dashboard
module Errors
DashboardProcessingError = Class.new(StandardError)
PanelNotFoundError = Class.new(StandardError)
LayoutError = Class.new(DashboardProcessingError)
MissingQueryError = Class.new(DashboardProcessingError)
PROCESSING_ERROR = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError
NOT_FOUND_ERROR = Gitlab::Template::Finders::RepoTemplateFinder::FileNotFoundError
def handle_errors(error)
case error
when PROCESSING_ERROR
when DashboardProcessingError
error(error.message, :unprocessable_entity)
when NOT_FOUND_ERROR
error("#{dashboard_path} could not be found.", :not_found)
......
......@@ -28,7 +28,7 @@ module Gitlab
# @param options - y_label [String] Y-Axis label of
# a panel. Used by embedded dashboards.
# @return [Hash]
def find(project, user, environment, options = {})
def find(project, user, environment = nil, options = {})
service_for(options)
.new(project, user, options.merge(environment: environment))
.get_dashboard
......
......@@ -21,10 +21,10 @@ module Gitlab
Stages::Sorter
].freeze
def initialize(project, environment, dashboard)
def initialize(project, dashboard, params)
@project = project
@environment = environment
@dashboard = dashboard
@params = params
end
# Returns a new dashboard hash with the results of
......@@ -32,7 +32,7 @@ module Gitlab
def process(insert_project_metrics:)
@dashboard.deep_symbolize_keys.tap do |dashboard|
sequence(insert_project_metrics).each do |stage|
stage.new(@project, @environment, dashboard).transform!
stage.new(@project, dashboard, @params).transform!
end
end
end
......
......@@ -48,3 +48,5 @@ module Gitlab
end
end
end
Gitlab::Metrics::Dashboard::ServiceSelector.prepend_if_ee('EE::Gitlab::Metrics::Dashboard::ServiceSelector')
......@@ -7,15 +7,12 @@ module Gitlab
class BaseStage
include Gitlab::Metrics::Dashboard::Defaults
DashboardProcessingError = Class.new(StandardError)
LayoutError = Class.new(DashboardProcessingError)
attr_reader :project, :dashboard, :params
attr_reader :project, :environment, :dashboard
def initialize(project, environment, dashboard)
def initialize(project, dashboard, params)
@project = project
@environment = environment
@dashboard = dashboard
@params = params
end
# Entry-point to the stage
......@@ -26,15 +23,15 @@ module Gitlab
protected
def missing_panel_groups!
raise LayoutError.new('Top-level key :panel_groups must be an array')
raise Errors::LayoutError.new('Top-level key :panel_groups must be an array')
end
def missing_panels!
raise LayoutError.new('Each "panel_group" must define an array :panels')
raise Errors::LayoutError.new('Each "panel_group" must define an array :panels')
end
def missing_metrics!
raise LayoutError.new('Each "panel" must define an array :metrics')
raise Errors::LayoutError.new('Each "panel" must define an array :metrics')
end
def for_metrics
......
......@@ -5,9 +5,9 @@ module Gitlab
module Dashboard
module Stages
class EndpointInserter < BaseStage
MissingQueryError = Class.new(DashboardProcessingError)
def transform!
raise Errors::DashboardProcessingError.new('Environment is required for Stages::EndpointInserter') unless params[:environment]
for_metrics do |metric|
metric[:prometheus_endpoint_path] = endpoint_for_metric(metric)
end
......@@ -18,7 +18,7 @@ module Gitlab
def endpoint_for_metric(metric)
Gitlab::Routing.url_helpers.prometheus_api_project_environment_path(
project,
environment,
params[:environment],
proxy_path: query_type(metric),
query: query_for_metric(metric)
)
......@@ -31,7 +31,7 @@ module Gitlab
def query_for_metric(metric)
query = metric[query_type(metric)]
raise MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query
raise Errors::MissingQueryError.new('Each "metric" must define one of :query or :query_range') unless query
query
end
......
......@@ -8,7 +8,7 @@ describe Gitlab::Metrics::Dashboard::Processor do
let(:dashboard_yml) { YAML.load_file('spec/fixtures/lib/gitlab/metrics/dashboard/sample_dashboard.yml') }
describe 'process' do
let(:process_params) { [project, environment, dashboard_yml] }
let(:process_params) { [project, dashboard_yml, { environment: environment }] }
let(:dashboard) { described_class.new(*process_params).process(insert_project_metrics: true) }
it 'includes a path for the prometheus endpoint with each metric' do
......@@ -67,7 +67,7 @@ describe Gitlab::Metrics::Dashboard::Processor do
shared_examples_for 'errors with message' do |expected_message|
it 'raises a DashboardLayoutError' do
error_class = Gitlab::Metrics::Dashboard::Stages::BaseStage::DashboardProcessingError
error_class = Gitlab::Metrics::Dashboard::Errors::DashboardProcessingError
expect { dashboard }.to raise_error(error_class, expected_message)
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment