Merge branch 'osw-add-redis-metrics-to-web-requests' into 'master'

Add metrics for Redis usage during web requests

See merge request gitlab-org/gitlab!32605

changelogs/unreleased/osw-add-redis-metrics-to-web-requests.yml

+---
+title: Add metrics for Redis usage during web requests
+merge_request: 32605
+author:
+type: added

config/initializers/zz_metrics.rb

@@ -147,6 +147,7 @@ if Gitlab::Metrics.enabled? && !Rails.env.test? && !(Rails.env.development? && d
   Gitlab::Application.configure do |config|
     config.middleware.use(Gitlab::Metrics::RackMiddleware)
     config.middleware.use(Gitlab::Middleware::RailsQueueDuration)
+    config.middleware.use(Gitlab::Metrics::RedisRackMiddleware)
   end
 
   Sidekiq.configure_server do |config|

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -92,6 +92,8 @@ The following metrics are available:
 | `gitlab_view_rendering_duration_seconds`                       | Histogram |                   10.2 | Duration for views (histogram)                                                                      | `controller`, `action`, `view`                      |
 | `http_requests_total`                                          | Counter   |                    9.4 | Rack request count                                                                                  | `method`                                            |
 | `http_request_duration_seconds`                                | Histogram |                    9.4 | HTTP response time from rack middleware                                                             | `method`, `status`                                  |
+| `http_redis_requests_duration_seconds`                         | Histogram |                   13.1 | Redis requests duration during web transactions                                                     | `controller`, `action`                              |
+| `http_redis_requests_total`                                    | Counter   |                   13.1 | Redis requests count during web transactions                                                        | `controller`, `action`                              |
 | `pipelines_created_total`                                      | Counter   |                    9.4 | Counter of pipelines created                                                                        |                                                     |
 | `rack_uncaught_errors_total`                                   | Counter   |                    9.4 | Rack connections handling uncaught errors count                                                     |                                                     |
 | `user_session_logins_total`                                    | Counter   |                    9.4 | Counter of how many users have logged in                                                            |                                                     |

lib/gitlab/metrics/redis_rack_middleware.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Metrics
+    # Rack middleware for tracking Redis metrics from Grape and Web requests.
+    class RedisRackMiddleware
+      def initialize(app)
+        @app = app
+      end
+
+      def call(env)
+        transaction = Gitlab::Metrics.current_transaction
+
+        @app.call(env)
+      ensure
+        record_metrics(transaction) if transaction
+      end
+
+      private
+
+      def record_metrics(transaction)
+        labels = transaction.labels
+        query_time = Gitlab::Instrumentation::Redis.query_time
+        request_count = Gitlab::Instrumentation::Redis.get_request_count
+
+        Gitlab::Metrics.counter(:http_redis_requests_total,
+                                'Amount of calls to Redis servers during web requests',
+                                Gitlab::Metrics::Transaction::BASE_LABELS).increment(labels, request_count)
+
+        Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds,
+                                  'Query time for Redis servers during web requests',
+                                  Gitlab::Metrics::Transaction::BASE_LABELS,
+                                  Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS).observe(labels, query_time)
+      end
+    end
+  end
+end

spec/lib/gitlab/metrics/redis_rack_middleware_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Metrics::RedisRackMiddleware do
+  let(:app) { double(:app) }
+  let(:middleware) { described_class.new(app) }
+  let(:env) { {} }
+  let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) }
+
+  before do
+    allow(app).to receive(:call).with(env).and_return('wub wub')
+  end
+
+  describe '#call' do
+    context 'when metrics are disabled' do
+      before do
+        allow(Gitlab::Metrics).to receive(:current_transaction).and_return(nil)
+      end
+
+      it 'calls the app' do
+        expect(middleware.call(env)).to eq('wub wub')
+      end
+
+      it 'does not record metrics' do
+        expect(Gitlab::Metrics).not_to receive(:counter)
+        expect(Gitlab::Metrics).not_to receive(:histogram)
+
+        middleware.call(env)
+      end
+    end
+
+    context 'when metrics are enabled' do
+      let(:counter) { double(Prometheus::Client::Counter, increment: nil) }
+      let(:histogram) { double(Prometheus::Client::Histogram, observe: nil) }
+      let(:redis_query_time) { 0.1 }
+      let(:redis_requests_count) { 2 }
+
+      before do
+        allow(Gitlab::Instrumentation::Redis).to receive(:query_time) { redis_query_time }
+        allow(Gitlab::Instrumentation::Redis).to receive(:get_request_count) { redis_requests_count }
+
+        allow(Gitlab::Metrics).to receive(:counter)
+          .with(:http_redis_requests_total,
+                an_instance_of(String),
+                Gitlab::Metrics::Transaction::BASE_LABELS)
+          .and_return(counter)
+
+        allow(Gitlab::Metrics).to receive(:histogram)
+          .with(:http_redis_requests_duration_seconds,
+                an_instance_of(String),
+                Gitlab::Metrics::Transaction::BASE_LABELS,
+                Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS)
+          .and_return(histogram)
+
+        allow(Gitlab::Metrics).to receive(:current_transaction).and_return(transaction)
+      end
+
+      it 'calls the app' do
+        expect(middleware.call(env)).to eq('wub wub')
+      end
+
+      it 'records redis metrics' do
+        expect(counter).to receive(:increment).with(transaction.labels, redis_requests_count)
+        expect(histogram).to receive(:observe).with(transaction.labels, redis_query_time)
+
+        middleware.call(env)
+      end
+
+      it 'records redis metrics if an error is raised' do
+        expect(counter).to receive(:increment).with(transaction.labels, redis_requests_count)
+        expect(histogram).to receive(:observe).with(transaction.labels, redis_query_time)
+
+        allow(app).to receive(:call).with(env).and_raise(StandardError)
+
+        expect { middleware.call(env) }.to raise_error(StandardError)
+      end
+    end
+  end
+end