Commit 74c81fb3 authored by Kirstie Cook's avatar Kirstie Cook Committed by Imre Farkas

Add rake file to generate sample metrics

Interpolate metric queries

Use yml for sample metric output to easily convert hash

Create a file for each metric

Replace metrics endpoint if using sample data

Need to create sample_metrics directory to run test

Moveing logic out of the controller

Insert live endpoint

Add specs for SampleMetricsService

Cleaning up controller specs

Return 404 instead of 500 when file is not found
parent 925e1dd2
# frozen_string_literal: true
class Projects::Environments::SampleMetricsController < Projects::ApplicationController
def query
result = Metrics::SampleMetricsService.new(params[:identifier]).query
if result
render json: { "status": "success", "data": { "resultType": "matrix", "result": result } }
else
render_404
end
end
end
# frozen_string_literal: true
module Metrics
class SampleMetricsService
DIRECTORY = "sample_metrics"
attr_reader :identifier
def initialize(identifier)
@identifier = identifier
end
def query
return unless identifier && File.exist?(file_location)
YAML.load_file(File.expand_path(file_location, __dir__))
end
private
def file_location
sanitized_string = identifier.gsub(/[^0-9A-Za-z_]/, '')
File.join(Rails.root, DIRECTORY, "#{sanitized_string}.yml")
end
end
end
---
title: Genereate a set of sample prometheus metrics and route to the sample metrics
when enabled
merge_request: 19987
author:
type: added
...@@ -232,6 +232,8 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do ...@@ -232,6 +232,8 @@ constraints(::Constraints::ProjectUrlConstrainer.new) do
get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', format: false get '/terminal.ws/authorize', to: 'environments#terminal_websocket_authorize', format: false
get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy', as: :prometheus_api get '/prometheus/api/v1/*proxy_path', to: 'environments/prometheus_api#proxy', as: :prometheus_api
get '/sample_metrics', to: 'environments/sample_metrics#query' if ENV['USE_SAMPLE_METRICS']
end end
collection do collection do
......
# Generate Sample Prometheus Data
This command will run Prometheus queries for each of the metrics of a specific environment
for a default time interval of 7 days ago to now. The results of each of query are stored
under a `sample_metrics` directory as a yaml file named by the metric's `identifier`.
When the environmental variable `USE_SAMPLE_METRICS` is set, the Prometheus API query is
re-routed to `Projects::Environments::SampleMetricsController` which loads the appropriate
data set if it is present within the `sample_metrics` directory.
- This command requires an id from an Environment with an available Prometheus installation.
**Example:**
```
bundle exec rake gitlab:generate_sample_prometheus_data[21]
```
...@@ -16,12 +16,20 @@ module Gitlab ...@@ -16,12 +16,20 @@ module Gitlab
private private
def endpoint_for_metric(metric) def endpoint_for_metric(metric)
Gitlab::Routing.url_helpers.prometheus_api_project_environment_path( if ENV['USE_SAMPLE_METRICS']
project, Gitlab::Routing.url_helpers.sample_metrics_project_environment_path(
params[:environment], project,
proxy_path: query_type(metric), params[:environment],
query: query_for_metric(metric) identifier: metric[:id]
) )
else
Gitlab::Routing.url_helpers.prometheus_api_project_environment_path(
project,
params[:environment],
proxy_path: query_type(metric),
query: query_for_metric(metric)
)
end
end end
def query_type(metric) def query_type(metric)
......
namespace :gitlab do
desc "GitLab | Generate Sample Prometheus Data"
task :generate_sample_prometheus_data, [:environment_id] => :gitlab_environment do |_, args|
environment = Environment.find(args[:environment_id])
metrics = PrometheusMetric.where(project_id: [environment.project.id, nil])
query_variables = Gitlab::Prometheus::QueryVariables.call(environment)
sample_metrics_directory_name = Metrics::SampleMetricsService::DIRECTORY
FileUtils.mkdir_p(sample_metrics_directory_name)
metrics.each do |metric|
query = metric.query % query_variables
result = environment.prometheus_adapter.prometheus_client.query_range(query, start: 7.days.ago)
next unless metric.identifier
File.write("#{sample_metrics_directory_name}/#{metric.identifier}.yml", result.to_yaml)
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Projects::Environments::SampleMetricsController do
include StubENV
let_it_be(:project) { create(:project) }
let_it_be(:environment) { create(:environment, project: project) }
let_it_be(:user) { create(:user) }
before(:context) do
RSpec::Mocks.with_temporary_scope do
stub_env('USE_SAMPLE_METRICS', 'true')
Rails.application.reload_routes!
end
end
after(:context) do
Rails.application.reload_routes!
end
before do
project.add_reporter(user)
sign_in(user)
end
describe 'GET #query' do
context 'when the file is not found' do
before do
get :query, params: environment_params
end
it 'returns a 404' do
expect(response).to have_gitlab_http_status(:not_found)
end
end
context 'when the sample data is found' do
before do
allow_next_instance_of(Metrics::SampleMetricsService) do |service|
allow(service).to receive(:query).and_return([])
end
get :query, params: environment_params
end
it 'returns JSON with a message and a 200 status code' do
expect(json_response.keys).to contain_exactly('status', 'data')
expect(response).to have_gitlab_http_status(:ok)
end
end
end
private
def environment_params(params = {})
{
id: environment.id.to_s,
namespace_id: project.namespace.full_path,
project_id: project.name,
identifier: 'sample_metric_query_result'
}.merge(params)
end
end
---
- metric: {}
values:
- - 1573560714.209
- '0.02361297607421875'
- - 1573560774.209
- '0.02361297607421875'
- - 1573560834.209
- '0.02362823486328125'
- - 1573560894.209
- '0.02361297607421875'
- - 1573560954.209
- '0.02385711669921875'
- - 1573561014.209
- '0.02361297607421875'
- - 1573561074.209
- '0.02361297607421875'
- - 1573561134.209
- '0.02362060546875'
- - 1573561194.209
- '0.02362060546875'
- - 1573561254.209
- '0.02362060546875'
- - 1573561314.209
- '0.02362060546875'
- - 1573561374.209
- '0.023624420166015625'
- - 1573561434.209
- '0.023651123046875'
- - 1573561494.209
- '0.02362060546875'
- - 1573561554.209
- '0.0236358642578125'
- - 1573561614.209
- '0.02362060546875'
- - 1573561674.209
- '0.02362060546875'
- - 1573561734.209
- '0.02362060546875'
- - 1573561794.209
- '0.02362060546875'
- - 1573561854.209
- '0.02362060546875'
- - 1573561914.209
- '0.023651123046875'
- - 1573561974.209
- '0.02362060546875'
- - 1573562034.209
- '0.02362060546875'
- - 1573562094.209
- '0.02362060546875'
- - 1573562154.209
- '0.02362060546875'
- - 1573562214.209
- '0.023624420166015625'
- - 1573562274.209
- '0.02362060546875'
- - 1573562334.209
- '0.023868560791015625'
- - 1573562394.209
- '0.02374267578125'
- - 1573562454.209
- '0.02362060546875'
- - 1573562514.209
- '0.02362060546875'
- - 1573562574.209
- '0.02362060546875'
- - 1573562634.209
- '0.02362060546875'
- - 1573562694.209
- '0.023639678955078125'
- - 1573562754.209
- '0.0236358642578125'
- - 1573562814.209
- '0.02362060546875'
- - 1573562874.209
- '0.0236358642578125'
- - 1573562934.209
- '0.023651123046875'
- - 1573562994.209
- '0.02362060546875'
- - 1573563054.209
- '0.023624420166015625'
- - 1573563114.209
- '0.02362060546875'
- - 1573563174.209
- '0.02362060546875'
- - 1573563234.209
- '0.02362060546875'
- - 1573563294.209
- '0.02362060546875'
- - 1573563354.209
- '0.02362060546875'
- - 1573563414.209
- '0.023651123046875'
- - 1573563474.209
- '0.023651123046875'
- - 1573563534.209
- '0.023651123046875'
- - 1573563594.209
- '0.023773193359375'
- - 1573563654.209
- '0.023681640625'
- - 1573563714.209
- '0.023895263671875'
- - 1573563774.209
- '0.023651123046875'
- - 1573563834.209
- '0.023651123046875'
- - 1573563894.209
- '0.023651123046875'
- - 1573563954.209
- '0.0236663818359375'
- - 1573564014.209
- '0.023651123046875'
- - 1573564074.209
- '0.023681640625'
- - 1573564134.209
- '0.0236663818359375'
- - 1573564194.209
- '0.0236663818359375'
- - 1573564254.209
- '0.023651123046875'
- - 1573564314.209
- '0.023651123046875'
- - 1573564374.209
- '0.023651123046875'
- - 1573564434.209
- '0.023773193359375'
- - 1573564494.209
- '0.023651123046875'
- - 1573564554.209
- '0.023681640625'
- - 1573564614.209
- '0.023773193359375'
- - 1573564674.209
- '0.023651123046875'
- - 1573564734.209
- '0.023651123046875'
- - 1573564794.209
- '0.023651123046875'
- - 1573564854.209
- '0.023651123046875'
- - 1573564914.209
- '0.023651123046875'
- - 1573564974.209
- '0.023651123046875'
- - 1573565034.209
- '0.023651123046875'
- - 1573565094.209
- '0.023895263671875'
\ No newline at end of file
# frozen_string_literal: true
require 'spec_helper'
describe Metrics::SampleMetricsService do
describe 'query' do
subject { described_class.new(identifier).query }
context 'when the file is not found' do
let(:identifier) { nil }
it { is_expected.to be_nil }
end
context 'when the file is found' do
let(:identifier) { 'sample_metric_query_result' }
let(:source) { File.join(Rails.root, 'spec/fixtures/gitlab/sample_metrics', "#{identifier}.yml") }
let(:destination) { File.join(Rails.root, Metrics::SampleMetricsService::DIRECTORY, "#{identifier}.yml") }
around do |example|
FileUtils.mkdir_p(Metrics::SampleMetricsService::DIRECTORY)
FileUtils.cp(source, destination)
example.run
ensure
FileUtils.rm(destination)
end
subject { described_class.new(identifier).query }
it 'loads data from the sample file correctly' do
expect(subject).to eq(YAML.load_file(source))
end
end
context 'when the identifier is for a path outside of sample_metrics' do
let(:identifier) { '../config/secrets' }
it { is_expected.to be_nil }
end
end
end
# frozen_string_literal: true
require 'rake_helper'
describe 'gitlab:generate_sample_prometheus_data rake task' do
let(:cluster) { create(:cluster, :provided_by_user, :project) }
let(:environment) { create(:environment, project: cluster.project) }
let(:sample_query_file) { File.join(Rails.root, Metrics::SampleMetricsService::DIRECTORY, 'test_query_result.yml') }
let!(:metric) { create(:prometheus_metric, project: cluster.project, identifier: 'test_query_result') }
around do |example|
example.run
ensure
FileUtils.rm(sample_query_file)
end
it 'creates the file correctly' do
Rake.application.rake_require 'tasks/gitlab/generate_sample_prometheus_data'
allow(Environment).to receive(:find).and_return(environment)
allow(environment).to receive_message_chain(:prometheus_adapter, :prometheus_client, :query_range) { sample_query_result }
run_rake_task('gitlab:generate_sample_prometheus_data', [environment.id])
expect(File.exist?(sample_query_file)).to be true
query_file_content = YAML.load_file(sample_query_file)
expect(query_file_content).to eq(sample_query_result)
end
end
def sample_query_result
file = File.join(Rails.root, 'spec/fixtures/gitlab/sample_metrics', 'sample_metric_query_result.yml')
YAML.load_file(File.expand_path(file, __dir__))
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment