Skip to content

G
gitlab-ce
Commit 71b97fb2 authored by Kirstie Cook's avatar Kirstie Cook Committed by Rémy Coutable
Browse files
Options

Add range to sample prometheus data 

Add sample prometheus data fixture
parent 9a6f5e58
Show whitespace changes
Inline Side-by-side
Showing with 369 additions and 163 deletions
app/controllers/projects/environments/sample_metrics_controller.rb
View file @ 71b97fb2
...@@ -2,7 +2,7 @@ ...@@ -2,7 +2,7 @@
class Projects::Environments::SampleMetricsController < Projects::ApplicationController class Projects::Environments::SampleMetricsController < Projects::ApplicationController
def query def query
result = Metrics::SampleMetricsService.new(params[:identifier]).query result = Metrics::SampleMetricsService.new(params[:identifier], range_start: params[:start], range_end: params[:end]).query
if result if result
render json: { "status": "success", "data": { "resultType": "matrix", "result": result } } render json: { "status": "success", "data": { "resultType": "matrix", "result": result } }
......
app/services/metrics/sample_metrics_service.rb
View file @ 71b97fb2
...@@ -4,16 +4,17 @@ module Metrics ...@@ -4,16 +4,17 @@ module Metrics
class SampleMetricsService class SampleMetricsService
DIRECTORY = "sample_metrics" DIRECTORY = "sample_metrics"
attr_reader :identifier attr_reader :identifier, :range_minutes
def initialize(identifier) def initialize(identifier, range_start:, range_end:)
@identifier = identifier @identifier = identifier
@range_minutes = convert_range_minutes(range_start, range_end)
end end
def query def query
return unless identifier && File.exist?(file_location) return unless identifier && File.exist?(file_location)
YAML.load_file(File.expand_path(file_location, __dir__)) query_interval
end end
private private
...@@ -22,5 +23,14 @@ module Metrics ...@@ -22,5 +23,14 @@ module Metrics
sanitized_string = identifier.gsub(/[^0-9A-Za-z_]/, '') sanitized_string = identifier.gsub(/[^0-9A-Za-z_]/, '')
File.join(Rails.root, DIRECTORY, "#{sanitized_string}.yml") File.join(Rails.root, DIRECTORY, "#{sanitized_string}.yml")
end end
def query_interval
result = YAML.load_file(File.expand_path(file_location, __dir__))
result[range_minutes]
end
def convert_range_minutes(range_start, range_end)
((range_end.to_time - range_start.to_time) / 1.minute).to_i
end
end end
end end
changelogs/unreleased/generate-sample-metrics-intervals.yml 0 → 100644
View file @ 71b97fb2
---
title: Generate Prometheus sample metrics over pre-set intervals
merge_request: 22066
author:
type: added
doc/raketasks/generate_sample_prometheus_data.md
View file @ 71b97fb2
# Generate Sample Prometheus Data # Generate Sample Prometheus Data
This command will run Prometheus queries for each of the metrics of a specific environment This command will run Prometheus queries for each of the metrics of a specific environment
for a default time interval of 7 days ago to now. The results of each of query are stored for a series of time intervals: 30 minutes, 3 hours, 8 hours, 24 hours, 72 hours, and 7 days
under a `sample_metrics` directory as a yaml file named by the metric's `identifier`. to now. The results of each of query are stored under a `sample_metrics` directory as a yaml
When the environmental variable `USE_SAMPLE_METRICS` is set, the Prometheus API query is file named by the metric's `identifier`. When the environmental variable `USE_SAMPLE_METRICS`
re-routed to `Projects::Environments::SampleMetricsController` which loads the appropriate is set, the Prometheus API query is re-routed to `Projects::Environments::SampleMetricsController`
data set if it is present within the `sample_metrics` directory. which loads the appropriate data set if it is present within the `sample_metrics` directory.
- This command requires an id from an Environment with an available Prometheus installation. - This command requires an id from an Environment with an available Prometheus installation.
......
lib/tasks/gitlab/generate_sample_prometheus_data.rake
View file @ 71b97fb2
...@@ -8,12 +8,17 @@ namespace :gitlab do ...@@ -8,12 +8,17 @@ namespace :gitlab do
sample_metrics_directory_name = Metrics::SampleMetricsService::DIRECTORY sample_metrics_directory_name = Metrics::SampleMetricsService::DIRECTORY
FileUtils.mkdir_p(sample_metrics_directory_name) FileUtils.mkdir_p(sample_metrics_directory_name)
sample_metrics_intervals = [30.minutes, 180.minutes, 8.hours, 24.hours, 72.hours, 7.days]
metrics.each do |metric| metrics.each do |metric|
query = metric.query % query_variables query = metric.query % query_variables
result = environment.prometheus_adapter.prometheus_client.query_range(query, start: 7.days.ago)
next unless metric.identifier next unless metric.identifier
result = sample_metrics_intervals.each_with_object({}) do |interval, memo|
memo[interval.to_i / 60] = environment.prometheus_adapter.prometheus_client.query_range(query, start: interval.ago)
end
File.write("#{sample_metrics_directory_name}/#{metric.identifier}.yml", result.to_yaml) File.write("#{sample_metrics_directory_name}/#{metric.identifier}.yml", result.to_yaml)
end end
end end
......
spec/controllers/projects/environments/sample_metrics_controller_spec.rb
View file @ 71b97fb2
...@@ -58,7 +58,9 @@ describe Projects::Environments::SampleMetricsController do ...@@ -58,7 +58,9 @@ describe Projects::Environments::SampleMetricsController do
id: environment.id.to_s, id: environment.id.to_s,
namespace_id: project.namespace.full_path, namespace_id: project.namespace.full_path,
project_id: project.name, project_id: project.name,
identifier: 'sample_metric_query_result' identifier: 'sample_metric_query_result',
start: '2019-12-02T23:31:45.000Z',
end: '2019-12-03T00:01:45.000Z'
}.merge(params) }.merge(params)
end end
end end
spec/fixtures/gitlab/sample_metrics/sample_metric_query_result.yml
View file @ 71b97fb2
--- ---
30:
- metric: {} - metric: {}
values: values:
- - 1573560714.209 - - 1576719533.248
- '0.02361297607421875' - '0.0006172414678571515'
- - 1573560774.209 - - 1576719593.248
- '0.02361297607421875' - '0.0006189408976190352'
- - 1573560834.209 - - 1576719653.248
- '0.02362823486328125' - '0.0006182154988094691'
- - 1573560894.209 - - 1576719713.248
- '0.02361297607421875' - '0.0006194998404763076'
- - 1573560954.209 - - 1576719773.248
- '0.02385711669921875' - '0.0006194687678569856'
- - 1573561014.209 - - 1576719833.248
- '0.02361297607421875' - '0.0006171203535713976'
- - 1573561074.209 - - 1576719893.248
- '0.02361297607421875' - '0.0006244061773808577'
- - 1573561134.209 - - 1576719953.248
- '0.02362060546875' - '0.0006170288511561634'
- - 1573561194.209 - - 1576720013.248
- '0.02362060546875' - '0.0006243750281248557'
- - 1573561254.209 - - 1576720073.248
- '0.02362060546875' - '0.0006152456571427256'
- - 1573561314.209 - - 1576720133.248
- '0.02362060546875' - '0.0006215679095237733'
- - 1573561374.209 - - 1576720193.248
- '0.023624420166015625' - '0.0006218523571429083'
- - 1573561434.209 - - 1576720253.248
- '0.023651123046875' - '0.0006200312440475792'
- - 1573561494.209 - - 1576720313.248
- '0.02362060546875' - '0.0006214166202382676'
- - 1573561554.209 - - 1576720373.248
- '0.0236358642578125' - '0.0006152486976191084'
- - 1573561614.209 - - 1576720433.248
- '0.02362060546875' - '0.0006136406750000235'
- - 1573561674.209 - - 1576720493.248
- '0.02362060546875' - '0.0006135999154761997'
- - 1573561734.209 - - 1576720553.248
- '0.02362060546875' - '0.0006126559190475756'
- - 1573561794.209 - - 1576720613.248
- '0.02362060546875' - '0.0006153160392857769'
- - 1573561854.209 - - 1576720673.248
- '0.02362060546875' - '0.0006146447178572262'
- - 1573561914.209 - - 1576720733.248
- '0.023651123046875' - '0.0006146970476189988'
- - 1573561974.209 - - 1576720793.248
- '0.02362060546875' - '0.0006219259035715042'
- - 1573562034.209 - - 1576720853.248
- '0.02362060546875' - '0.0006111198750001481'
- - 1573562094.209 - - 1576720913.248
- '0.02362060546875' - '0.0006169941035715337'
- - 1573562154.209 - - 1576720973.248
- '0.02362060546875' - '0.0006102626761905379'
- - 1573562214.209 - - 1576721033.248
- '0.023624420166015625' - '0.0006163839964285346'
- - 1573562274.209 180:
- '0.02362060546875' - metric: {}
- - 1573562334.209 values:
- '0.023868560791015625' - - 1576719533.248
- - 1573562394.209 - '0.0006172414678571515'
- '0.02374267578125' - - 1576719593.248
- - 1573562454.209 - '0.0006189408976190352'
- '0.02362060546875' - - 1576719653.248
- - 1573562514.209 - '0.0006182154988094691'
- '0.02362060546875' - - 1576719713.248
- - 1573562574.209 - '0.0006194998404763076'
- '0.02362060546875' - - 1576719773.248
- - 1573562634.209 - '0.0006194687678569856'
- '0.02362060546875' - - 1576719833.248
- - 1573562694.209 - '0.0006171203535713976'
- '0.023639678955078125' - - 1576719893.248
- - 1573562754.209 - '0.0006244061773808577'
- '0.0236358642578125' - - 1576719953.248
- - 1573562814.209 - '0.0006170288511561634'
- '0.02362060546875' - - 1576720013.248
- - 1573562874.209 - '0.0006243750281248557'
- '0.0236358642578125' - - 1576720073.248
- - 1573562934.209 - '0.0006152456571427256'
- '0.023651123046875' - - 1576720133.248
- - 1573562994.209 - '0.0006215679095237733'
- '0.02362060546875' - - 1576720193.248
- - 1573563054.209 - '0.0006218523571429083'
- '0.023624420166015625' - - 1576720253.248
- - 1573563114.209 - '0.0006200312440475792'
- '0.02362060546875' - - 1576720313.248
- - 1573563174.209 - '0.0006214166202382676'
- '0.02362060546875' - - 1576720373.248
- - 1573563234.209 - '0.0006152486976191084'
- '0.02362060546875' - - 1576720433.248
- - 1573563294.209 - '0.0006136406750000235'
- '0.02362060546875' - - 1576720493.248
- - 1573563354.209 - '0.0006135999154761997'
- '0.02362060546875' - - 1576720553.248
- - 1573563414.209 - '0.0006126559190475756'
- '0.023651123046875' - - 1576720613.248
- - 1573563474.209 - '0.0006153160392857769'
- '0.023651123046875' - - 1576720673.248
- - 1573563534.209 - '0.0006146447178572262'
- '0.023651123046875' - - 1576720733.248
- - 1573563594.209 - '0.0006146970476189988'
- '0.023773193359375' - - 1576720793.248
- - 1573563654.209 - '0.0006219259035715042'
- '0.023681640625' - - 1576720853.248
- - 1573563714.209 - '0.0006111198750001481'
- '0.023895263671875' - - 1576720913.248
- - 1573563774.209 - '0.0006169941035715337'
- '0.023651123046875' - - 1576720973.248
- - 1573563834.209 - '0.0006102626761905379'
- '0.023651123046875' - - 1576721033.248
- - 1573563894.209 - '0.0006163839964285346'
- '0.023651123046875' 480:
- - 1573563954.209 - metric: {}
- '0.0236663818359375' values:
- - 1573564014.209 - - 1576719533.248
- '0.023651123046875' - '0.0006172414678571515'
- - 1573564074.209 - - 1576719593.248
- '0.023681640625' - '0.0006189408976190352'
- - 1573564134.209 - - 1576719653.248
- '0.0236663818359375' - '0.0006182154988094691'
- - 1573564194.209 - - 1576719713.248
- '0.0236663818359375' - '0.0006194998404763076'
- - 1573564254.209 - - 1576719773.248
- '0.023651123046875' - '0.0006194687678569856'
- - 1573564314.209 - - 1576719833.248
- '0.023651123046875' - '0.0006171203535713976'
- - 1573564374.209 - - 1576719893.248
- '0.023651123046875' - '0.0006244061773808577'
- - 1573564434.209 - - 1576719953.248
- '0.023773193359375' - '0.0006170288511561634'
- - 1573564494.209 - - 1576720013.248
- '0.023651123046875' - '0.0006243750281248557'
- - 1573564554.209 - - 1576720073.248
- '0.023681640625' - '0.0006152456571427256'
- - 1573564614.209 - - 1576720133.248
- '0.023773193359375' - '0.0006215679095237733'
- - 1573564674.209 - - 1576720193.248
- '0.023651123046875' - '0.0006218523571429083'
- - 1573564734.209 - - 1576720253.248
- '0.023651123046875' - '0.0006200312440475792'
- - 1573564794.209 - - 1576720313.248
- '0.023651123046875' - '0.0006214166202382676'
- - 1573564854.209 - - 1576720373.248
- '0.023651123046875' - '0.0006152486976191084'
- - 1573564914.209 - - 1576720433.248
- '0.023651123046875' - '0.0006136406750000235'
- - 1573564974.209 - - 1576720493.248
- '0.023651123046875' - '0.0006135999154761997'
- - 1573565034.209 - - 1576720553.248
- '0.023651123046875' - '0.0006126559190475756'
- - 1573565094.209 - - 1576720613.248
- '0.023895263671875' - '0.0006153160392857769'
\ No newline at end of file - - 1576720673.248
- '0.0006146447178572262'
- - 1576720733.248
- '0.0006146970476189988'
- - 1576720793.248
- '0.0006219259035715042'
- - 1576720853.248
- '0.0006111198750001481'
- - 1576720913.248
- '0.0006169941035715337'
- - 1576720973.248
- '0.0006102626761905379'
- - 1576721033.248
- '0.0006163839964285346'
1440:
- metric: {}
values:
- - 1576719533.248
- '0.0006172414678571515'
- - 1576719593.248
- '0.0006189408976190352'
- - 1576719653.248
- '0.0006182154988094691'
- - 1576719713.248
- '0.0006194998404763076'
- - 1576719773.248
- '0.0006194687678569856'
- - 1576719833.248
- '0.0006171203535713976'
- - 1576719893.248
- '0.0006244061773808577'
- - 1576719953.248
- '0.0006170288511561634'
- - 1576720013.248
- '0.0006243750281248557'
- - 1576720073.248
- '0.0006152456571427256'
- - 1576720133.248
- '0.0006215679095237733'
- - 1576720193.248
- '0.0006218523571429083'
- - 1576720253.248
- '0.0006200312440475792'
- - 1576720313.248
- '0.0006214166202382676'
- - 1576720373.248
- '0.0006152486976191084'
- - 1576720433.248
- '0.0006136406750000235'
- - 1576720493.248
- '0.0006135999154761997'
- - 1576720553.248
- '0.0006126559190475756'
- - 1576720613.248
- '0.0006153160392857769'
- - 1576720673.248
- '0.0006146447178572262'
- - 1576720733.248
- '0.0006146970476189988'
- - 1576720793.248
- '0.0006219259035715042'
- - 1576720853.248
- '0.0006111198750001481'
- - 1576720913.248
- '0.0006169941035715337'
- - 1576720973.248
- '0.0006102626761905379'
- - 1576721033.248
- '0.0006163839964285346'
4320:
- metric: {}
values:
- - 1576719533.248
- '0.0006172414678571515'
- - 1576719593.248
- '0.0006189408976190352'
- - 1576719653.248
- '0.0006182154988094691'
- - 1576719713.248
- '0.0006194998404763076'
- - 1576719773.248
- '0.0006194687678569856'
- - 1576719833.248
- '0.0006171203535713976'
- - 1576719893.248
- '0.0006244061773808577'
- - 1576719953.248
- '0.0006170288511561634'
- - 1576720013.248
- '0.0006243750281248557'
- - 1576720073.248
- '0.0006152456571427256'
- - 1576720133.248
- '0.0006215679095237733'
- - 1576720193.248
- '0.0006218523571429083'
- - 1576720253.248
- '0.0006200312440475792'
- - 1576720313.248
- '0.0006214166202382676'
- - 1576720373.248
- '0.0006152486976191084'
- - 1576720433.248
- '0.0006136406750000235'
- - 1576720493.248
- '0.0006135999154761997'
- - 1576720553.248
- '0.0006126559190475756'
- - 1576720613.248
- '0.0006153160392857769'
- - 1576720673.248
- '0.0006146447178572262'
- - 1576720733.248
- '0.0006146970476189988'
- - 1576720793.248
- '0.0006219259035715042'
- - 1576720853.248
- '0.0006111198750001481'
- - 1576720913.248
- '0.0006169941035715337'
- - 1576720973.248
- '0.0006102626761905379'
- - 1576721033.248
- '0.0006163839964285346'
10080:
- metric: {}
values:
- - 1576719533.248
- '0.0006172414678571515'
- - 1576719593.248
- '0.0006189408976190352'
- - 1576719653.248
- '0.0006182154988094691'
- - 1576719713.248
- '0.0006194998404763076'
- - 1576719773.248
- '0.0006194687678569856'
- - 1576719833.248
- '0.0006171203535713976'
- - 1576719893.248
- '0.0006244061773808577'
- - 1576719953.248
- '0.0006170288511561634'
- - 1576720013.248
- '0.0006243750281248557'
- - 1576720073.248
- '0.0006152456571427256'
- - 1576720133.248
- '0.0006215679095237733'
- - 1576720193.248
- '0.0006218523571429083'
- - 1576720253.248
- '0.0006200312440475792'
- - 1576720313.248
- '0.0006214166202382676'
- - 1576720373.248
- '0.0006152486976191084'
- - 1576720433.248
- '0.0006136406750000235'
- - 1576720493.248
- '0.0006135999154761997'
- - 1576720553.248
- '0.0006126559190475756'
- - 1576720613.248
- '0.0006153160392857769'
- - 1576720673.248
- '0.0006146447178572262'
- - 1576720733.248
- '0.0006146970476189988'
- - 1576720793.248
- '0.0006219259035715042'
- - 1576720853.248
- '0.0006111198750001481'
- - 1576720913.248
- '0.0006169941035715337'
- - 1576720973.248
- '0.0006102626761905379'
- - 1576721033.248
- '0.0006163839964285346'
\ No newline at end of file
spec/services/metrics/sample_metrics_service_spec.rb
View file @ 71b97fb2
...@@ -4,7 +4,10 @@ require 'spec_helper' ...@@ -4,7 +4,10 @@ require 'spec_helper'
describe Metrics::SampleMetricsService do describe Metrics::SampleMetricsService do
describe 'query' do describe 'query' do
subject { described_class.new(identifier).query } let(:range_start) { '2019-12-02T23:31:45.000Z' }
let(:range_end) { '2019-12-03T00:01:45.000Z' }
subject { described_class.new(identifier, range_start: range_start, range_end: range_end).query }
context 'when the file is not found' do context 'when the file is not found' do
let(:identifier) { nil } let(:identifier) { nil }
...@@ -26,10 +29,10 @@ describe Metrics::SampleMetricsService do ...@@ -26,10 +29,10 @@ describe Metrics::SampleMetricsService do
FileUtils.rm(destination) FileUtils.rm(destination)
end end
subject { described_class.new(identifier).query } subject { described_class.new(identifier, range_start: range_start, range_end: range_end).query }
it 'loads data from the sample file correctly' do it 'loads data from the sample file correctly' do
expect(subject).to eq(YAML.load_file(source)) expect(subject).to eq(YAML.load_file(source)[30])
end end
end end
......
spec/tasks/gitlab/generate_sample_prometheus_data_spec.rb
View file @ 71b97fb2
...@@ -17,7 +17,7 @@ describe 'gitlab:generate_sample_prometheus_data rake task' do ...@@ -17,7 +17,7 @@ describe 'gitlab:generate_sample_prometheus_data rake task' do
it 'creates the file correctly' do it 'creates the file correctly' do
Rake.application.rake_require 'tasks/gitlab/generate_sample_prometheus_data' Rake.application.rake_require 'tasks/gitlab/generate_sample_prometheus_data'
allow(Environment).to receive(:find).and_return(environment) allow(Environment).to receive(:find).and_return(environment)
allow(environment).to receive_message_chain(:prometheus_adapter, :prometheus_client, :query_range) { sample_query_result } allow(environment).to receive_message_chain(:prometheus_adapter, :prometheus_client, :query_range) { sample_query_result[30] }
run_rake_task('gitlab:generate_sample_prometheus_data', [environment.id]) run_rake_task('gitlab:generate_sample_prometheus_data', [environment.id])
expect(File.exist?(sample_query_file)).to be true expect(File.exist?(sample_query_file)).to be true
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7