Add metrics for tracking stale secondaries for merge requests

To fix the merge requests failing to match due to replication lag
(https://gitlab.com/gitlab-org/gitlab/-/issues/247857) we added a
mechanism to track the WAL pointer of the merge request to decide
whether to use the secondary or fall back to the primary. This commit
adds metrics to better understand how often this fallback is occurring.

ee/changelogs/unreleased/sh-add-matching-merge-request-metrics.yml

+---
+title: Add metrics for tracking stale secondaries for merge requests
+merge_request: 44813
+author:
+type: other

ee/lib/ee/gitlab/checks/matching_merge_request.rb

@@ -6,6 +6,9 @@ module EE
       module MatchingMergeRequest
         extend ::Gitlab::Utils::Override
 
+        TOTAL_METRIC = :gitlab_merge_request_match_total
+        STALE_METRIC = :gitlab_merge_request_match_stale_secondary
+
         override :match?
         def match?
           return super unless ::Feature.enabled?(:matching_merge_request_db_sync)
@@ -24,9 +27,44 @@ module EE
           # sessions, replication lag could erroneously cause step 5 to
           # report no matching merge requests. To avoid this, we check
           # the write location to ensure the replica can make this query.
+          track_session_metrics do
             ::Gitlab::Database::LoadBalancing::Sticking.unstick_or_continue_sticking(:project, @project.id) # rubocop:disable Gitlab/ModuleWithInstanceVariables
+          end
+
           super
         end
+
+        private
+
+        def track_session_metrics
+          before = ::Gitlab::Database::LoadBalancing::Session.current.use_primary?
+
+          yield
+
+          after = ::Gitlab::Database::LoadBalancing::Session.current.use_primary?
+
+          increment_attempt_count
+
+          if !before && after
+            increment_stale_secondary_count
+          end
+        end
+
+        def increment_attempt_count
+          total_counter.increment
+        end
+
+        def increment_stale_secondary_count
+          stale_counter.increment
+        end
+
+        def total_counter
+          @total_counter ||= ::Gitlab::Metrics.counter(TOTAL_METRIC, 'Total number of merge request match attempts')
+        end
+
+        def stale_counter
+          @stale_counter ||= ::Gitlab::Metrics.counter(STALE_METRIC, 'Total number of merge request match attempts with lagging secondary')
+        end
       end
     end
   end

ee/spec/lib/gitlab/checks/matching_merge_request_spec.rb

@@ -15,22 +15,51 @@ RSpec.describe Gitlab::Checks::MatchingMergeRequest do
         target_branch: target_branch,
         in_progress_merge_commit_sha: newrev)
     end
+    let(:total_counter) { subject.send(:total_counter) }
+    let(:stale_counter) { subject.send(:stale_counter) }
 
     subject { described_class.new(newrev, target_branch, project) }
 
     context 'with load balancing disabled', :request_store, :redis do
       before do
         expect(::Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(false)
+        expect(::Gitlab::Database::LoadBalancing::Sticking).not_to receive(:unstick_or_continue_sticking)
       end
 
       it 'does not attempt to stick to primary' do
-        expect(::Gitlab::Database::LoadBalancing::Sticking).not_to receive(:unstick_or_continue_sticking)
-
         expect(subject.match?).to be true
       end
+
+      it 'increments no counters' do
+        expect { subject.match? }
+          .to change { total_counter.get }.by(0)
+          .and change { stale_counter.get }.by(0)
+      end
     end
 
     context 'with load balancing enabled', :request_store, :redis do
+      let(:session) { ::Gitlab::Database::LoadBalancing::Session.current }
+
+      context 'on secondary that has caught up to primary' do
+        before do
+          allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?).and_return(true)
+          expect(::Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true)
+          expect(::Gitlab::Database::LoadBalancing::Sticking).to receive(:unstick_or_continue_sticking).and_call_original
+        end
+
+        it 'continues to use the secondary' do
+          expect(session.use_primary?).to be false
+          expect(subject.match?).to be true
+        end
+
+        it 'only increments total counter' do
+          expect { subject.match? }
+            .to change { total_counter.get }.by(1)
+            .and change { stale_counter.get }.by(0)
+        end
+      end
+
+      context 'on secondary behind primary' do
         before do
           allow(::Gitlab::Database::LoadBalancing::Sticking).to receive(:all_caught_up?).and_return(false)
         end
@@ -38,24 +67,39 @@ RSpec.describe Gitlab::Checks::MatchingMergeRequest do
         context 'with feature flag disabled' do
           before do
             stub_feature_flags(matching_merge_request_db_sync: false)
-        end
-
-        it 'does not check load balancing state' do
             expect(::Gitlab::Database::LoadBalancing).not_to receive(:enable?)
             expect(::Gitlab::Database::LoadBalancing::Sticking).not_to receive(:unstick_or_continue_sticking)
+          end
 
+          it 'does not check load balancing state' do
             expect(subject.match?).to be true
           end
+
+          it 'increments no counters' do
+            expect { subject.match? }
+              .to change { total_counter.get }.by(0)
+              .and change { stale_counter.get }.by(0)
+          end
         end
 
-      it 'sticks to the primary' do
-        session = ::Gitlab::Database::LoadBalancing::Session.current
+        context 'load balancing enabled' do
+          before do
             expect(::Gitlab::Database::LoadBalancing).to receive(:enable?).and_return(true)
             expect(::Gitlab::Database::LoadBalancing::Sticking).to receive(:unstick_or_continue_sticking).and_call_original
+          end
 
+          it 'sticks to the primary' do
             expect(subject.match?).to be true
             expect(session.use_primary?).to be true
           end
+
+          it 'increments both total and stale counters' do
+            expect { subject.match? }
+              .to change { total_counter.get }.by(1)
+              .and change { stale_counter.get }.by(1)
+          end
+        end
+      end
     end
   end
 end