Merge branch '333777-add-cached-wal-count-metric' into 'master'

Add db_primary_wal_cached_count and db_replica_wal_cached_count metrics

See merge request gitlab-org/gitlab!64210

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -98,6 +98,8 @@ The following metrics are available:
 | `gitlab_transaction_db_write_count_total`                        | Counter     | 13.1    | Counter for total number of write SQL calls                                                                           | `controller`, `action`                                    |
 | `gitlab_transaction_db_cached_count_total`                       | Counter     | 13.1    | Counter for total number of cached SQL calls                                                                          | `controller`, `action`                                    |
 | `gitlab_transaction_db_<role>_cached_count_total`                | Counter     | 13.1    | Counter for total number of cached SQL calls, grouped by database roles (primary/replica)                             | `controller`, `action`                                    |
+| `gitlab_transaction_db_<role>_wal_count_total`                   | Counter     | 14.0    | Counter for total number of WAL (write ahead log location) queries, grouped by database roles (primary/replica)       | `controller`, `action`                                    |
+| `gitlab_transaction_db_<role>_wal_cached_count_total`            | Counter     | 14.1    | Counter for total number of cached WAL (write ahead log location) queries, grouped by database roles (primary/replica)| `controller`, `action`                                    |
 | `http_elasticsearch_requests_duration_seconds` **(PREMIUM)**     | Histogram   | 13.1    | Elasticsearch requests duration during web transactions                                                               | `controller`, `action`                                    |
 | `http_elasticsearch_requests_total` **(PREMIUM)**                | Counter     | 13.1    | Elasticsearch requests count during web transactions                                                                  | `controller`, `action`                                    |
 | `pipelines_created_total`                                        | Counter     | 9.4     | Counter of pipelines created                                                                                          |                                                           |

lib/gitlab/metrics/subscribers/active_record.rb

@@ -15,8 +15,8 @@ module Gitlab
         TRANSACTION_DURATION_BUCKET = [0.1, 0.25, 1].freeze
 
         DB_LOAD_BALANCING_COUNTERS = %i{
-          db_replica_count db_replica_cached_count db_replica_wal_count
-          db_primary_count db_primary_cached_count db_primary_wal_count
+          db_replica_count db_replica_cached_count db_replica_wal_count db_replica_wal_cached_count
+          db_primary_count db_primary_cached_count db_primary_wal_count db_primary_wal_cached_count
         }.freeze
         DB_LOAD_BALANCING_DURATIONS = %i{db_primary_duration_s db_replica_duration_s}.freeze
 
@@ -91,9 +91,14 @@ module Gitlab
         end
 
         def increment_db_role_counters(db_role, payload)
+          cached = cached_query?(payload)
           increment("db_#{db_role}_count".to_sym)
-          increment("db_#{db_role}_cached_count".to_sym) if cached_query?(payload)
-          increment("db_#{db_role}_wal_count".to_sym) if !cached_query?(payload) && wal_command?(payload)
+          increment("db_#{db_role}_cached_count".to_sym) if cached
+
+          if wal_command?(payload)
+            increment("db_#{db_role}_wal_count".to_sym)
+            increment("db_#{db_role}_wal_cached_count".to_sym) if cached
+          end
         end
 
         def observe_db_role_duration(db_role, event)

spec/initializers/lograge_spec.rb

@@ -200,6 +200,8 @@ RSpec.describe 'lograge', type: :request do
         %w[
           db_primary_wal_count
           db_replica_wal_count
+          db_primary_wal_cached_count
+          db_replica_wal_cached_count
           db_replica_count
           db_replica_cached_count
           db_primary_count

spec/lib/gitlab/instrumentation_helper_spec.rb

@@ -133,7 +133,9 @@ RSpec.describe Gitlab::InstrumentationHelper do
                                    db_primary_count: 0,
                                    db_primary_cached_count: 0,
                                    db_primary_wal_count: 0,
-                                   db_replica_wal_count: 0)
+                                   db_replica_wal_count: 0,
+                                   db_primary_wal_cached_count: 0,
+                                   db_replica_wal_cached_count: 0)
       end
     end
 
@@ -150,7 +152,9 @@ RSpec.describe Gitlab::InstrumentationHelper do
                                    db_primary_count: 0,
                                    db_primary_cached_count: 0,
                                    db_primary_wal_count: 0,
-                                   db_replica_wal_count: 0)
+                                   db_replica_wal_count: 0,
+                                   db_primary_wal_cached_count: 0,
+                                   db_replica_wal_cached_count: 0)
       end
     end
   end

spec/lib/gitlab/metrics/subscribers/active_record_spec.rb

@@ -183,6 +183,8 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActiveRecord do
         'SQL' | 'UPDATE users SET admin = true WHERE id = 10' | true | true | false | false
         'SQL' | 'SELECT pg_current_wal_insert_lsn()::text AS location' | true | false | false | true
         'SQL' | 'SELECT pg_last_wal_replay_lsn()::text AS location' | true | false | false | true
+        'CACHE' | 'SELECT pg_current_wal_insert_lsn()::text AS location' | true | false | true | true
+        'CACHE' | 'SELECT pg_last_wal_replay_lsn()::text AS location' | true | false | true | true
         'CACHE' | 'SELECT * FROM users WHERE id = 10' | true | false | true | false
         'SCHEMA' | "SELECT attr.attname FROM pg_attribute attr INNER JOIN pg_constraint cons ON attr.attrelid = cons.conrelid AND attr.attnum = any(cons.conkey) WHERE cons.contype = 'p' AND cons.conrelid = '\"projects\"'::regclass" | false | false | false | false
         nil | 'BEGIN' | false | false | false | false

spec/lib/gitlab/sidekiq_logging/structured_logger_spec.rb

@@ -314,7 +314,9 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do
             'db_primary_cached_count' => 0,
             'db_primary_wal_count' => 0,
             'db_primary_duration_s' => a_value > 0,
-            "db_primary_#{dbname}_duration_s" => a_value > 0
+            "db_primary_#{dbname}_duration_s" => a_value > 0,
+            'db_primary_wal_cached_count' => 0,
+            'db_replica_wal_cached_count' => 0
           )
         end
 
@@ -336,6 +338,8 @@ RSpec.describe Gitlab::SidekiqLogging::StructuredLogger do
             'db_primary_count' => 0,
             'db_primary_cached_count' => 0,
             'db_primary_wal_count' => 0,
+            'db_primary_wal_cached_count' => 0,
+            'db_replica_wal_cached_count' => 0,
             'db_primary_duration_s' => 0
           )
         end

spec/support/shared_examples/metrics/active_record_subscriber_shared_examples.rb

@@ -19,6 +19,8 @@ RSpec.shared_examples 'store ActiveRecord info in RequestStore' do |db_role|
             db_replica_count:  0,
             db_replica_duration_s:  0.0,
             db_primary_wal_count: record_wal_query ? 1 : 0,
+            db_primary_wal_cached_count: record_wal_query && record_cached_query ? 1 : 0,
+            db_replica_wal_cached_count: 0,
             db_replica_wal_count: 0
           }
           expected[:"db_primary_#{::Gitlab::Database.dbname(connection)}_duration_s"] = 0.002 if record_query
@@ -34,6 +36,8 @@ RSpec.shared_examples 'store ActiveRecord info in RequestStore' do |db_role|
             db_replica_count:  record_query ? 1 : 0,
             db_replica_duration_s:  record_query ? 0.002 : 0,
             db_replica_wal_count: record_wal_query ? 1 : 0,
+            db_replica_wal_cached_count: record_wal_query && record_cached_query ? 1 : 0,
+            db_primary_wal_cached_count: 0,
             db_primary_wal_count: 0
           }
           expected[:"db_replica_#{::Gitlab::Database.dbname(connection)}_duration_s"] = 0.002 if record_query
@@ -91,7 +95,10 @@ RSpec.shared_examples 'record ActiveRecord metrics in a metrics transaction' do
     end
 
     if record_wal_query
-      expect(transaction).to receive(:increment).with("gitlab_transaction_db_#{db_role}_wal_count_total".to_sym, 1) if db_role
+      if db_role
+        expect(transaction).to receive(:increment).with("gitlab_transaction_db_#{db_role}_wal_count_total".to_sym, 1)
+        expect(transaction).to receive(:increment).with("gitlab_transaction_db_#{db_role}_wal_cached_count_total".to_sym, 1) if record_cached_query
+      end
     else
       expect(transaction).not_to receive(:increment).with("gitlab_transaction_db_#{db_role}_wal_count_total".to_sym, 1) if db_role
     end