Merge branch '214279-es-instrumentation-server-metrics' into 'master'

Add Elasticsearch to Sidekiq ServerMetrics

Closes #214279

See merge request gitlab-org/gitlab!32937

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -126,10 +126,12 @@ configuration option in `gitlab.yml`. These metrics are served from the
 | `sidekiq_jobs_db_seconds`                      | Histogram | 12.9 | Seconds of DB time to run Sidekiq job                                                               | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` |
 | `sidekiq_jobs_gitaly_seconds`                  | Histogram | 12.9 | Seconds of Gitaly time to run Sidekiq job                                                           | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` |
 | `sidekiq_redis_requests_duration_seconds`      | Histogram | 13.1 | Duration in seconds that a Sidekiq job spent querying a Redis server                                | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` |
+| `sidekiq_elasticsearch_requests_duration_seconds`      | Histogram | 13.1 | Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server                                | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` |
 | `sidekiq_jobs_queue_duration_seconds`          | Histogram | 12.5 | Duration in seconds that a Sidekiq job was queued before being executed                             | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |
 | `sidekiq_jobs_failed_total`                    | Counter   | 12.2 | Sidekiq jobs failed                                                                                 | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |
 | `sidekiq_jobs_retried_total`                   | Counter   | 12.2 | Sidekiq jobs retried                                                                                | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |
 | `sidekiq_redis_requests_total`                 | Counter   | 13.1 | Redis requests during a Sidekiq job execution                                                       | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` |
+| `sidekiq_elasticsearch_requests_total`         | Counter   | 13.1 | Elasticsearch requests during a Sidekiq job execution                                                       | `queue`, `boundary`, `external_dependencies`, `feature_category`, `job_status`, `urgency` |
 | `sidekiq_running_jobs`                         | Gauge     | 12.2 | Number of Sidekiq jobs running                                                                      | `queue`, `boundary`, `external_dependencies`, `feature_category`, `urgency` |
 | `sidekiq_concurrency`                          | Gauge     | 12.5 | Maximum number of Sidekiq jobs                                                                      |                                                                   |
 | `geo_db_replication_lag_seconds`               | Gauge   | 10.2  | Database replication lag (seconds) | `url` |

ee/changelogs/unreleased/214279-es-instrumentation-server-metrics.yml

+---
+title: Add Elasticsearch to Sidekiq ServerMetrics
+merge_request: 32937
+author:
+type: added

lib/gitlab/sidekiq_middleware/server_metrics.rb

@@ -49,6 +49,8 @@ module Gitlab
           @metrics[:sidekiq_jobs_gitaly_seconds].observe(labels, get_gitaly_time(job))
           @metrics[:sidekiq_redis_requests_total].increment(labels, get_redis_calls(job))
           @metrics[:sidekiq_redis_requests_duration_seconds].observe(labels, get_redis_time(job))
+          @metrics[:sidekiq_elasticsearch_requests_total].increment(labels, get_elasticsearch_calls(job))
+          @metrics[:sidekiq_elasticsearch_requests_duration_seconds].observe(labels, get_elasticsearch_time(job))
         end
       end
 
@@ -62,9 +64,11 @@ module Gitlab
           sidekiq_jobs_gitaly_seconds:             ::Gitlab::Metrics.histogram(:sidekiq_jobs_gitaly_seconds, 'Seconds of Gitaly time to run Sidekiq job', {}, SIDEKIQ_LATENCY_BUCKETS),
           sidekiq_jobs_queue_duration_seconds:     ::Gitlab::Metrics.histogram(:sidekiq_jobs_queue_duration_seconds, 'Duration in seconds that a Sidekiq job was queued before being executed', {}, SIDEKIQ_LATENCY_BUCKETS),
           sidekiq_redis_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_redis_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent requests a Redis server', {}, Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS),
+          sidekiq_elasticsearch_requests_duration_seconds: ::Gitlab::Metrics.histogram(:sidekiq_elasticsearch_requests_duration_seconds, 'Duration in seconds that a Sidekiq job spent in requests to an Elasticsearch server', {}, SIDEKIQ_LATENCY_BUCKETS),
           sidekiq_jobs_failed_total:               ::Gitlab::Metrics.counter(:sidekiq_jobs_failed_total, 'Sidekiq jobs failed'),
           sidekiq_jobs_retried_total:              ::Gitlab::Metrics.counter(:sidekiq_jobs_retried_total, 'Sidekiq jobs retried'),
           sidekiq_redis_requests_total:            ::Gitlab::Metrics.counter(:sidekiq_redis_requests_total, 'Redis requests during a Sidekiq job execution'),
+          sidekiq_elasticsearch_requests_total:    ::Gitlab::Metrics.counter(:sidekiq_elasticsearch_requests_total, 'Elasticsearch requests during a Sidekiq job execution'),
           sidekiq_running_jobs:                    ::Gitlab::Metrics.gauge(:sidekiq_running_jobs, 'Number of Sidekiq jobs running', {}, :all),
           sidekiq_concurrency:                     ::Gitlab::Metrics.gauge(:sidekiq_concurrency, 'Maximum number of Sidekiq jobs', {}, :all)
         }
@@ -82,6 +86,14 @@ module Gitlab
         job.fetch(:redis_calls, 0)
       end
 
+      def get_elasticsearch_time(job)
+        job.fetch(:elasticsearch_duration_s, 0)
+      end
+
+      def get_elasticsearch_calls(job)
+        job.fetch(:elasticsearch_calls, 0)
+      end
+
       def get_gitaly_time(job)
         job.fetch(:gitaly_duration_s, 0)
       end

spec/lib/gitlab/sidekiq_middleware/server_metrics_spec.rb

@@ -34,6 +34,8 @@ describe Gitlab::SidekiqMiddleware::ServerMetrics do
         let(:redis_requests_total) { double('redis calls total metric') }
         let(:running_jobs_metric) { double('running jobs metric') }
         let(:redis_seconds_metric) { double('redis seconds metric') }
+        let(:elasticsearch_seconds_metric) { double('elasticsearch seconds metric') }
+        let(:elasticsearch_requests_total) { double('elasticsearch calls total metric') }
 
         before do
           allow(Gitlab::Metrics).to receive(:histogram).with(:sidekiq_jobs_queue_duration_seconds, anything, anything, anything).and_return(queue_duration_seconds)
@@ -42,9 +44,11 @@ describe Gitlab::SidekiqMiddleware::ServerMetrics do
           allow(Gitlab::Metrics).to receive(:histogram).with(:sidekiq_jobs_db_seconds, anything, anything, anything).and_return(db_seconds_metric)
           allow(Gitlab::Metrics).to receive(:histogram).with(:sidekiq_jobs_gitaly_seconds, anything, anything, anything).and_return(gitaly_seconds_metric)
           allow(Gitlab::Metrics).to receive(:histogram).with(:sidekiq_redis_requests_duration_seconds, anything, anything, anything).and_return(redis_seconds_metric)
+          allow(Gitlab::Metrics).to receive(:histogram).with(:sidekiq_elasticsearch_requests_duration_seconds, anything, anything, anything).and_return(elasticsearch_seconds_metric)
           allow(Gitlab::Metrics).to receive(:counter).with(:sidekiq_jobs_failed_total, anything).and_return(failed_total_metric)
           allow(Gitlab::Metrics).to receive(:counter).with(:sidekiq_jobs_retried_total, anything).and_return(retried_total_metric)
           allow(Gitlab::Metrics).to receive(:counter).with(:sidekiq_redis_requests_total, anything).and_return(redis_requests_total)
+          allow(Gitlab::Metrics).to receive(:counter).with(:sidekiq_elasticsearch_requests_total, anything).and_return(elasticsearch_requests_total)
           allow(Gitlab::Metrics).to receive(:gauge).with(:sidekiq_running_jobs, anything, {}, :all).and_return(running_jobs_metric)
           allow(Gitlab::Metrics).to receive(:gauge).with(:sidekiq_concurrency, anything, {}, :all).and_return(concurrency_metric)
 
@@ -76,6 +80,9 @@ describe Gitlab::SidekiqMiddleware::ServerMetrics do
           let(:redis_calls) { 2 }
           let(:redis_duration) { 0.01 }
 
+          let(:elasticsearch_calls) { 8 }
+          let(:elasticsearch_duration) { 0.54 }
+
           before do
             allow(subject).to receive(:get_thread_cputime).and_return(thread_cputime_before, thread_cputime_after)
             allow(Gitlab::Metrics::System).to receive(:monotonic_time).and_return(monotonic_time_before, monotonic_time_after)
@@ -86,14 +93,19 @@ describe Gitlab::SidekiqMiddleware::ServerMetrics do
             job[:redis_calls] = redis_calls
             job[:redis_duration_s] = redis_duration
 
+            job[:elasticsearch_calls] = elasticsearch_calls
+            job[:elasticsearch_duration_s] = elasticsearch_duration
+
             allow(running_jobs_metric).to receive(:increment)
             allow(redis_requests_total).to receive(:increment)
+            allow(elasticsearch_requests_total).to receive(:increment)
             allow(queue_duration_seconds).to receive(:observe)
             allow(user_execution_seconds_metric).to receive(:observe)
             allow(db_seconds_metric).to receive(:observe)
             allow(gitaly_seconds_metric).to receive(:observe)
             allow(completion_seconds_metric).to receive(:observe)
             allow(redis_seconds_metric).to receive(:observe)
+            allow(elasticsearch_seconds_metric).to receive(:observe)
           end
 
           it 'yields block' do
@@ -109,7 +121,9 @@ describe Gitlab::SidekiqMiddleware::ServerMetrics do
             expect(gitaly_seconds_metric).to receive(:observe).with(labels_with_job_status, gitaly_duration)
             expect(completion_seconds_metric).to receive(:observe).with(labels_with_job_status, monotonic_time_duration)
             expect(redis_seconds_metric).to receive(:observe).with(labels_with_job_status, redis_duration)
+            expect(elasticsearch_seconds_metric).to receive(:observe).with(labels_with_job_status, elasticsearch_duration)
             expect(redis_requests_total).to receive(:increment).with(labels_with_job_status, redis_calls)
+            expect(elasticsearch_requests_total).to receive(:increment).with(labels_with_job_status, elasticsearch_calls)
 
             subject.call(worker, job, :test) { nil }
           end