Refactor remaining metric usages

- Refactor Git Blob
- Refactor Highlight Cache
- Refactor Method Call
- Refactor Gitlab Database
- Fix Specs

config/initializers/zz_metrics.rb

@@ -199,7 +199,7 @@ if Gitlab::Metrics.enabled? && !Rails.env.test? && !(Rails.env.development? && d
       val = super
 
       if current_transaction = ::Gitlab::Metrics::Transaction.current
-        current_transaction.increment(:new_redis_connections, 1)
+        current_transaction.increment(:gitlab_transaction_new_redis_connections_total, 1)
       end
 
       val

lib/gitlab/database.rb

@@ -2,7 +2,6 @@
 
 module Gitlab
   module Database
-    include Gitlab::Metrics::Methods
 
     # Minimum PostgreSQL version requirement per documentation:
     # https://docs.gitlab.com/ee/install/requirements.html#postgresql-requirements
@@ -50,10 +49,6 @@ module Gitlab
     # It does not include the default public schema
     EXTRA_SCHEMAS = [DYNAMIC_PARTITIONS_SCHEMA, STATIC_PARTITIONS_SCHEMA].freeze
 
-    define_histogram :gitlab_database_transaction_seconds do
-      docstring "Time spent in database transactions, in seconds"
-    end
-
     def self.config
       ActiveRecord::Base.configurations[Rails.env]
     end
@@ -363,8 +358,11 @@ module Gitlab
     # observe_transaction_duration is called from ActiveRecordBaseTransactionMetrics.transaction and used to
     # record transaction durations.
     def self.observe_transaction_duration(duration_seconds)
-      labels = Gitlab::Metrics::Transaction.current&.labels || {}
-      gitlab_database_transaction_seconds.observe(labels, duration_seconds)
+      if current_transaction = ::Gitlab::Metrics::Transaction.current
+        current_transaction.observe(:gitlab_database_transaction_seconds, duration_seconds) do
+          docstring "Time spent in database transactions, in seconds"
+        end
+      end
     rescue Prometheus::Client::LabelSetValidator::LabelSetError => err
       # Ensure that errors in recording these metrics don't affect the operation of the application
       Rails.logger.error("Unable to observe database transaction duration: #{err}") # rubocop:disable Gitlab/RailsLogger

lib/gitlab/diff/highlight_cache.rb

@@ -3,7 +3,6 @@
 module Gitlab
   module Diff
     class HighlightCache
-      include Gitlab::Metrics::Methods
       include Gitlab::Utils::StrongMemoize
 
       EXPIRATION = 1.week
@@ -12,19 +11,6 @@ module Gitlab
       delegate :diffable,     to: :@diff_collection
       delegate :diff_options, to: :@diff_collection
 
-      define_histogram :gitlab_redis_diff_caching_memory_usage_bytes do
-        docstring 'Redis diff caching memory usage by key'
-        buckets [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000]
-      end
-
-      define_counter :gitlab_redis_diff_caching_hit do
-        docstring 'Redis diff caching hits'
-      end
-
-      define_counter :gitlab_redis_diff_caching_miss do
-        docstring 'Redis diff caching misses'
-      end
-
       def initialize(diff_collection)
         @diff_collection = diff_collection
       end
@@ -117,7 +103,10 @@ module Gitlab
 
       def record_memory_usage(memory_usage)
         if memory_usage
-          self.class.gitlab_redis_diff_caching_memory_usage_bytes.observe({}, memory_usage)
+          current_transaction&.observe(:gitlab_redis_diff_caching_memory_usage_bytes, memory_usage) do
+            docstring 'Redis diff caching memory usage by key'
+            buckets [100, 1_000, 10_000, 100_000, 1_000_000, 10_000_000]
+          end
         end
       end
 
@@ -206,6 +195,10 @@ module Gitlab
         #
         @diff_collection.raw_diff_files
       end
+
+      def current_transaction
+        ::Gitlab::Metrics::Transaction.current
+      end
     end
   end
 end

lib/gitlab/git/blob.rb

@@ -5,7 +5,6 @@ module Gitlab
     class Blob
       include Gitlab::BlobHelper
       include Gitlab::EncodingHelper
-      include Gitlab::Metrics::Methods
       extend Gitlab::Git::WrapsGitalyErrors
 
       # This number is the maximum amount of data that we want to display to
@@ -28,19 +27,6 @@ module Gitlab
       attr_accessor :size, :mode, :id, :commit_id, :loaded_size, :binary
       attr_writer :name, :path, :data
 
-      define_counter :gitlab_blob_truncated_true do
-        docstring 'blob.truncated? == true'
-      end
-
-      define_counter :gitlab_blob_truncated_false do
-        docstring 'blob.truncated? == false'
-      end
-
-      define_histogram :gitlab_blob_size do
-        docstring 'Gitlab::Git::Blob size'
-        buckets [1_000, 5_000, 10_000, 50_000, 100_000, 500_000, 1_000_000]
-      end
-
       class << self
         def find(repository, sha, path, limit: MAX_DATA_DISPLAY_SIZE)
           tree_entry(repository, sha, path, limit)
@@ -207,20 +193,31 @@ module Gitlab
       def record_metric_blob_size
         return unless size
 
-        self.class.gitlab_blob_size.observe({}, size)
+        current_transaction&.observe(:gitlab_blob_size, size) do
+          docstring 'Gitlab::Git::Blob size'
+          buckets [1_000, 5_000, 10_000, 50_000, 100_000, 500_000, 1_000_000]
+        end
       end
 
       def record_metric_truncated(bool)
         if bool
-          self.class.gitlab_blob_truncated_true.increment
+          current_transaction&.increment(:gitlab_blob_truncated_true) do
+            docstring 'blob.truncated? == true'
+          end
         else
-          self.class.gitlab_blob_truncated_false.increment
+          current_transaction&.increment(:gitlab_blob_truncated_false) do
+            docstring 'blob.truncated? == false'
+          end
         end
       end
 
       def has_lfs_version_key?
         !empty? && text_in_repo? && data.start_with?("version https://git-lfs.github.com/spec")
       end
+
+      def current_transaction
+        ::Gitlab::Metrics::Transaction.current
+      end
     end
   end
 end

lib/gitlab/gitaly_client.rb

@@ -8,8 +8,6 @@ require 'grpc/health/v1/health_services_pb'
 
 module Gitlab
   module GitalyClient
-    include Gitlab::Metrics::Methods
-
     class TooManyInvocationsError < StandardError
       attr_reader :call_site, :invocation_count, :max_call_stack
 
@@ -191,11 +189,6 @@ module Gitlab
       Gitlab::SafeRequestStore[:gitaly_query_time] += duration
     end
 
-    def self.current_transaction_labels
-      Gitlab::Metrics::Transaction.current&.labels || {}
-    end
-    private_class_method :current_transaction_labels
-
     # For some time related tasks we can't rely on `Time.now` since it will be
     # affected by Timecop in some tests, and the clock of some gitaly-related
     # components (grpc's c-core and gitaly server) use system time instead of

lib/gitlab/metrics.rb

@@ -3,7 +3,6 @@
 module Gitlab
   module Metrics
     include Gitlab::Metrics::Prometheus
-    include Gitlab::Metrics::Methods
 
     EXECUTION_MEASUREMENT_BUCKETS = [0.001, 0.01, 0.1, 1].freeze
 
@@ -81,25 +80,16 @@ module Gitlab
       real_time = (real_stop - real_start)
       cpu_time = cpu_stop - cpu_start
 
-      real_duration_seconds = fetch_histogram("gitlab_#{name}_real_duration_seconds".to_sym) do
+      trans.observe("gitlab_#{name}_real_duration_seconds".to_sym, real_time) do
         docstring "Measure #{name}"
-        base_labels Transaction::BASE_LABELS
         buckets EXECUTION_MEASUREMENT_BUCKETS
       end
 
-      real_duration_seconds.observe(trans.labels, real_time)
-
-      cpu_duration_seconds = fetch_histogram("gitlab_#{name}_cpu_duration_seconds".to_sym) do
+      trans.observe("gitlab_#{name}_cpu_duration_seconds".to_sym, cpu_time) do
         docstring "Measure #{name}"
-        base_labels Transaction::BASE_LABELS
         buckets EXECUTION_MEASUREMENT_BUCKETS
         with_feature "prometheus_metrics_measure_#{name}_cpu_duration"
       end
-      cpu_duration_seconds.observe(trans.labels, cpu_time)
-
-      trans.increment("#{name}_real_time", real_time.in_milliseconds, false)
-      trans.increment("#{name}_cpu_time", cpu_time.in_milliseconds, false)
-      trans.increment("#{name}_call_count", 1, false)
 
       retval
     end

lib/gitlab/metrics/elasticsearch_rack_middleware.rb

@@ -8,14 +8,6 @@ module Gitlab
 
       def initialize(app)
         @app = app
-
-        @requests_total_counter = Gitlab::Metrics.counter(:http_elasticsearch_requests_total,
-                                                         'Amount of calls to Elasticsearch servers during web requests',
-                                                         Gitlab::Metrics::Transaction::BASE_LABELS)
-        @requests_duration_histogram = Gitlab::Metrics.histogram(:http_elasticsearch_requests_duration_seconds,
-                                                                 'Query time for Elasticsearch servers during web requests',
-                                                                 Gitlab::Metrics::Transaction::BASE_LABELS,
-                                                                 HISTOGRAM_BUCKETS)
       end
 
       def call(env)
@@ -29,12 +21,16 @@ module Gitlab
       private
 
       def record_metrics(transaction)
-        labels = transaction.labels
         query_time = ::Gitlab::Instrumentation::ElasticsearchTransport.query_time
         request_count = ::Gitlab::Instrumentation::ElasticsearchTransport.get_request_count
 
-        @requests_total_counter.increment(labels, request_count)
-        @requests_duration_histogram.observe(labels, query_time)
+        transaction.increment(:http_elasticsearch_requests_total, request_count) do
+          docstring 'Amount of calls to Elasticsearch servers during web requests'
+        end
+        transaction.observe(:http_elasticsearch_requests_duration_seconds, query_time) do
+          docstring 'Query time for Elasticsearch servers during web requests'
+          buckets HISTOGRAM_BUCKETS
+        end
       end
     end
   end

lib/gitlab/metrics/method_call.rb

@@ -4,16 +4,7 @@ module Gitlab
   module Metrics
     # Class for tracking timing information about method calls
     class MethodCall
-      include Gitlab::Metrics::Methods
-      BASE_LABELS = { module: nil, method: nil }.freeze
-      attr_reader :real_time, :cpu_time, :call_count, :labels
-
-      define_histogram :gitlab_method_call_duration_seconds do
-        docstring 'Method calls real duration'
-        base_labels Transaction::BASE_LABELS.merge(BASE_LABELS)
-        buckets [0.01, 0.05, 0.1, 0.5, 1]
-        with_feature :prometheus_metrics_method_instrumentation
-      end
+      attr_reader :real_time, :cpu_time, :call_count, :labels, :transaction
 
       # name - The full name of the method (including namespace) such as
       #        `User#sign_in`.
@@ -42,8 +33,13 @@ module Gitlab
         @cpu_time += cpu_time
         @call_count += 1
 
-        if above_threshold?
-          self.class.gitlab_method_call_duration_seconds.observe(@transaction.labels.merge(labels), real_time)
+        if above_threshold? && transaction
+          transaction.observe(:gitlab_method_call_duration_seconds, real_time) do
+            docstring 'Method calls real duration'
+            base_labels @labels
+            buckets [0.01, 0.05, 0.1, 0.5, 1]
+            with_feature :prometheus_metrics_method_instrumentation
+          end
         end
 
         retval

lib/gitlab/metrics/methods.rb

@@ -69,62 +69,6 @@ module Gitlab
             raise ArgumentError, "uknown metric type #{type}"
           end
         end
-
-        # Fetch and/or initialize counter metric
-        # @param [Symbol] name
-        # @param [Hash] opts
-        def fetch_counter(name, opts = {}, &block)
-          fetch_metric(:counter, name, opts, &block)
-        end
-
-        # Fetch and/or initialize gauge metric
-        # @param [Symbol] name
-        # @param [Hash] opts
-        def fetch_gauge(name, opts = {}, &block)
-          fetch_metric(:gauge, name, opts, &block)
-        end
-
-        # Fetch and/or initialize histogram metric
-        # @param [Symbol] name
-        # @param [Hash] opts
-        def fetch_histogram(name, opts = {}, &block)
-          fetch_metric(:histogram, name, opts, &block)
-        end
-
-        # Fetch and/or initialize summary metric
-        # @param [Symbol] name
-        # @param [Hash] opts
-        def fetch_summary(name, opts = {}, &block)
-          fetch_metric(:summary, name, opts, &block)
-        end
-
-        # Define metric accessor method for a Counter
-        # @param [Symbol] name
-        # @param [Hash] opts
-        def define_counter(name, opts = {}, &block)
-          define_metric(:counter, name, opts, &block)
-        end
-
-        # Define metric accessor method for a Gauge
-        # @param [Symbol] name
-        # @param [Hash] opts
-        def define_gauge(name, opts = {}, &block)
-          define_metric(:gauge, name, opts, &block)
-        end
-
-        # Define metric accessor method for a Histogram
-        # @param [Symbol] name
-        # @param [Hash] opts
-        def define_histogram(name, opts = {}, &block)
-          define_metric(:histogram, name, opts, &block)
-        end
-
-        # Define metric accessor method for a Summary
-        # @param [Symbol] name
-        # @param [Hash] opts
-        def define_summary(name, opts = {}, &block)
-          define_metric(:summary, name, opts, &block)
-        end
       end
     end
   end

lib/gitlab/metrics/rack_middleware.rb

@@ -10,8 +10,7 @@ module Gitlab
 
       # env - A Hash containing Rack environment details.
       def call(env)
-        trans  = transaction_from_env(env)
-        retval = nil
+        trans = WebTransaction.new(env)
 
         begin
           retval = trans.run { @app.call(env) }
@@ -24,21 +23,6 @@ module Gitlab
 
         retval
       end
-
-      def transaction_from_env(env)
-        trans = WebTransaction.new(env)
-
-        trans.set(:request_uri, filtered_path(env), false)
-        trans.set(:request_method, env['REQUEST_METHOD'], false)
-
-        trans
-      end
-
-      private
-
-      def filtered_path(env)
-        ActionDispatch::Request.new(env).filtered_path.presence || env['REQUEST_URI']
-      end
     end
   end
 end

lib/gitlab/metrics/redis_rack_middleware.rb

@@ -6,14 +6,6 @@ module Gitlab
     class RedisRackMiddleware
       def initialize(app)
         @app = app
-
-        @requests_total_counter = Gitlab::Metrics.counter(:http_redis_requests_total,
-                                                          'Amount of calls to Redis servers during web requests',
-                                                          Gitlab::Metrics::Transaction::BASE_LABELS)
-        @requests_duration_histogram = Gitlab::Metrics.histogram(:http_redis_requests_duration_seconds,
-                                                                 'Query time for Redis servers during web requests',
-                                                                 Gitlab::Metrics::Transaction::BASE_LABELS,
-                                                                 Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS)
       end
 
       def call(env)
@@ -27,12 +19,16 @@ module Gitlab
       private
 
       def record_metrics(transaction)
-        labels = transaction.labels
         query_time = Gitlab::Instrumentation::Redis.query_time
         request_count = Gitlab::Instrumentation::Redis.get_request_count
 
-        @requests_total_counter.increment(labels, request_count)
-        @requests_duration_histogram.observe(labels, query_time)
+        transaction.increment(:http_redis_requests_total, request_count) do
+          docstring 'Amount of calls to Redis servers during web requests'
+        end
+        transaction.observe(:http_redis_requests_duration_seconds, query_time) do
+          docstring 'Query time for Redis servers during web requests'
+          buckets Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS
+        end
       end
     end
   end

lib/gitlab/metrics/subscribers/action_view.rb

@@ -5,14 +5,6 @@ module Gitlab
     module Subscribers
       # Class for tracking the rendering timings of views.
       class ActionView < ActiveSupport::Subscriber
-        include Gitlab::Metrics::Methods
-        define_histogram :gitlab_view_rendering_duration_seconds do
-          docstring 'View rendering time'
-          base_labels Transaction::BASE_LABELS.merge({ path: nil })
-          buckets [0.001, 0.01, 0.1, 1, 10.0]
-          with_feature :prometheus_metrics_view_instrumentation
-        end
-
         attach_to :action_view
 
         SERIES = 'views'
@@ -27,10 +19,13 @@ module Gitlab
 
         def track(event)
           tags = tags_for(event)
-
-          self.class.gitlab_view_rendering_duration_seconds.observe(current_transaction.labels.merge(tags), event.duration)
-
-          current_transaction.increment(:view_duration, event.duration)
+          current_transaction.observe(:gitlab_view_rendering_duration_seconds, event.duration) do
+            docstring 'View rendering time'
+            base_labels tags
+            buckets [0.001, 0.01, 0.1, 1, 10.0]
+            with_feature :prometheus_metrics_view_instrumentation
+          end
+          current_transaction.increment(:gitlab_transaction_view_duration_total, event.duration)
         end
 
         def relative_path(path)

lib/gitlab/metrics/subscribers/active_record.rb

@@ -5,7 +5,6 @@ module Gitlab
     module Subscribers
       # Class for tracking the total query duration of a transaction.
       class ActiveRecord < ActiveSupport::Subscriber
-        include Gitlab::Metrics::Methods
         attach_to :active_record
 
         IGNORABLE_SQL = %w{BEGIN COMMIT}.freeze
@@ -22,7 +21,9 @@ module Gitlab
           payload = event.payload
           return if payload[:name] == 'SCHEMA' || IGNORABLE_SQL.include?(payload[:sql])
 
-          self.class.gitlab_sql_duration_seconds.observe(current_transaction.labels, event.duration / 1000.0)
+          current_transaction.observe(:gitlab_sql_duration_seconds, event.duration / 1000.0) do
+            buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
+          end
 
           increment_db_counters(payload)
         end
@@ -37,12 +38,6 @@ module Gitlab
 
         private
 
-        define_histogram :gitlab_sql_duration_seconds do
-          docstring 'SQL time'
-          base_labels Transaction::BASE_LABELS
-          buckets [0.01, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
-        end
-
         def select_sql_command?(payload)
           payload[:sql].match(/\A((?!(.*[^\w'"](DELETE|UPDATE|INSERT INTO)[^\w'"])))(WITH.*)?(SELECT)((?!(FOR UPDATE|FOR SHARE)).)*$/i)
         end

lib/gitlab/metrics/subscribers/rails_cache.rb

@@ -14,11 +14,10 @@ module Gitlab
           return unless current_transaction
           return if event.payload[:super_operation] == :fetch
 
-          if event.payload[:hit]
-            current_transaction.increment(:cache_read_hit_count, 1, false)
-          else
-            metric_cache_misses_total.increment(current_transaction.labels)
-            current_transaction.increment(:cache_read_miss_count, 1, false)
+          unless event.payload[:hit]
+            current_transaction.increment(:gitlab_cache_misses_total, 1) do
+              docstring 'Cache read miss'
+            end
           end
         end
 
@@ -37,25 +36,31 @@ module Gitlab
         def cache_fetch_hit(event)
           return unless current_transaction
 
-          current_transaction.increment(:cache_read_hit_count, 1)
+          current_transaction.increment(:gitlab_transaction_cache_read_hit_count_total, 1)
         end
 
         def cache_generate(event)
           return unless current_transaction
 
-          metric_cache_misses_total.increment(current_transaction.labels)
-          current_transaction.increment(:cache_read_miss_count, 1)
+          current_transaction.increment(:gitlab_cache_misses_total, 1) do
+            docstring 'Cache read miss'
+          end
+          current_transaction.increment(:gitlab_transaction_cache_read_miss_count_total, 1)
         end
 
         def observe(key, duration)
           return unless current_transaction
 
-          metric_cache_operations_total.increment(current_transaction.labels.merge({ operation: key }))
-          metric_cache_operation_duration_seconds.observe({ operation: key }, duration / 1000.0)
-          current_transaction.increment(:cache_duration, duration, false)
-          current_transaction.increment(:cache_count, 1, false)
-          current_transaction.increment("cache_#{key}_duration".to_sym, duration, false)
-          current_transaction.increment("cache_#{key}_count".to_sym, 1, false)
+          labels = { operation: key }
+          current_transaction.increment(:gitlab_cache_operations_total, 1) do
+            docstring 'Cache operations'
+            base_labels labels
+          end
+          current_transaction.observe(:gitlab_cache_operation_duration_seconds, duration / 1000.0) do
+            docstring 'Cache access time'
+            buckets [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0]
+            base_labels labels
+          end
         end
 
         private
@@ -63,31 +68,6 @@ module Gitlab
         def current_transaction
           Transaction.current
         end
-
-        def metric_cache_operations_total
-          @metric_cache_operations_total ||= ::Gitlab::Metrics.counter(
-            :gitlab_cache_operations_total,
-            'Cache operations',
-            Transaction::BASE_LABELS
-          )
-        end
-
-        def metric_cache_operation_duration_seconds
-          @metric_cache_operation_duration_seconds ||= ::Gitlab::Metrics.histogram(
-            :gitlab_cache_operation_duration_seconds,
-            'Cache access time',
-            {},
-            [0.00001, 0.0001, 0.001, 0.01, 0.1, 1.0]
-          )
-        end
-
-        def metric_cache_misses_total
-          @metric_cache_misses_total ||= ::Gitlab::Metrics.counter(
-            :gitlab_cache_misses_total,
-            'Cache read miss',
-            Transaction::BASE_LABELS
-          )
-        end
       end
     end
   end

lib/gitlab/metrics/transaction.rb

@@ -13,13 +13,31 @@ module Gitlab
 
       THREAD_KEY = :_gitlab_metrics_transaction
 
+      SMALL_BUCKETS = [0.1, 0.25, 0.5, 1.0, 2.5, 5.0].freeze
+      BIG_BUCKETS = [100, 1000, 10000, 100000, 1000000, 10000000].freeze
+
       # The series to store events (e.g. Git pushes) in.
       EVENT_SERIES = 'events'
 
       attr_reader :method
 
-      def self.current
-        Thread.current[THREAD_KEY]
+      class << self
+        def current
+          Thread.current[THREAD_KEY]
+        end
+
+        def prometheus_metric(name, type, &block)
+          fetch_metric(type, name) do
+            # set default metric options
+            docstring "#{name.to_s.humanize} #{type}"
+            multiprocess_mode :livesum if type == :gauge
+            buckets SMALL_BUCKETS if type == :histogram
+
+            evaluate(&block)
+            # always filter sensitive labels and merge with base ones
+            base_labels base_labels.without(*FILTERED_LABELS).merge(BASE_LABELS)
+          end
+        end
       end
 
       def initialize
@@ -56,9 +74,11 @@ module Gitlab
         @memory_after = System.memory_usage_rss
         @finished_at = System.monotonic_time
 
-        self.class.gitlab_transaction_cputime_seconds.observe(labels, thread_cpu_duration)
-        self.class.gitlab_transaction_duration_seconds.observe(labels, duration)
-        self.class.gitlab_transaction_allocated_memory_bytes.observe(labels, allocated_memory * 1024.0)
+        observe(:cputime_seconds, thread_cpu_duration)
+        observe(:duration_seconds, duration)
+        observe(:allocated_memory_bytes, allocated_memory * 1024.0) do
+          buckets BIG_BUCKETS
+        end
 
         Thread.current[THREAD_KEY] = nil
       end
@@ -71,8 +91,12 @@ module Gitlab
       # event_name - The name of the event (e.g. "git_push").
       # tags - A set of tags to attach to the event.
       def add_event(event_name, tags = {})
-        filtered_tags = filter_tags(tags)
-        self.class.transaction_metric(event_name, :counter, prefix: 'event_', tags: filtered_tags).increment(filtered_tags.merge(labels))
+        event_name = "gitlab_transaction_event_#{event_name}_total".to_sym
+        metric = self.class.prometheus_metric(event_name, :counter) do
+          base_labels tags
+        end
+
+        metric.increment(tags.without(*FILTERED_LABELS).merge(labels))
       end
 
       # Returns a MethodCall object for the given name.
@@ -84,52 +108,65 @@ module Gitlab
         method
       end
 
-      def increment(name, value, use_prometheus = true)
-        self.class.transaction_metric(name, :counter).increment(labels, value) if use_prometheus
-      end
-
-      def set(name, value, use_prometheus = true)
-        self.class.transaction_metric(name, :gauge).set(labels, value) if use_prometheus
-      end
-
-      def labels
-        BASE_LABELS
-      end
+      # Increment counter metric
+      #
+      # It will initialize the metric if metric is not found
+      #
+      # block - if provided can be used to initialize metric with custom options (docstring, labels, with_feature)
+      #
+      # Example:
+      # ```
+      # transaction.increment(:mestric_name, 1, { docstring: 'Custom title', base_labels: {sane: 'yes'} } ) do
+      #
+      # transaction.increment(:mestric_name, 1) do
+      #   docstring 'Custom title'
+      #   base_labels { sane: 'yes' }
+      # end
+      # ```
+      def increment(name, value = 1, &block)
+        counter = self.class.prometheus_metric(name, :counter, &block)
 
-      define_histogram :gitlab_transaction_cputime_seconds do
-        docstring 'Transaction thread cputime'
-        base_labels BASE_LABELS
-        buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
+        counter.increment(counter.base_labels, value)
       end
 
-      define_histogram :gitlab_transaction_duration_seconds do
-        docstring 'Transaction duration'
-        base_labels BASE_LABELS
-        buckets [0.1, 0.25, 0.5, 1.0, 2.5, 5.0]
-      end
+      # Set gauge metric
+      #
+      # It will initialize the metric if metric is not found
+      #
+      # block - if provided, it can be used to initialize metric with custom options (docstring, labels, with_feature, multiprocess_mode)
+      #
+      # Example:
+      # ```
+      # transaction.set(:mestric_name, 1) do
+      #   multiprocess_mode :all
+      # end
+      # ```
+      def set(name, value, &block)
+        gauge = self.class.prometheus_metric(name, :gauge, &block)
 
-      define_histogram :gitlab_transaction_allocated_memory_bytes do
-        docstring 'Transaction allocated memory bytes'
-        base_labels BASE_LABELS
-        buckets [100, 1000, 10000, 100000, 1000000, 10000000]
+        gauge.set(gauge.base_labels, value)
       end
 
-      def self.transaction_metric(name, type, prefix: nil, tags: {})
-        metric_name = "gitlab_transaction_#{prefix}#{name}_total".to_sym
-        fetch_metric(type, metric_name) do
-          docstring "Transaction #{prefix}#{name} #{type}"
-          base_labels tags.merge(BASE_LABELS)
+      # Observe histogram metric
+      #
+      # It will initialize the metric if metric is not found
+      #
+      # block - if provided, it can be used to initialize metric with custom options (docstring, labels, with_feature, buckets)
+      #
+      # Example:
+      # ```
+      # transaction.observe(:mestric_name, 1) do
+      #   buckets [100, 1000, 10000, 100000, 1000000, 10000000]
+      # end
+      # ```
+      def observe(name, value, &block)
+        histogram = self.class.prometheus_metric(name, :histogram, &block)
 
-          if type == :gauge
-            multiprocess_mode :livesum
-          end
-        end
+        histogram.observe(histogram.base_labels, value)
       end
 
-      private
-
-      def filter_tags(tags)
-        tags.without(*FILTERED_LABELS)
+      def labels
+        BASE_LABELS
       end
     end
   end

lib/gitlab/middleware/rails_queue_duration.rb

@@ -19,25 +19,17 @@ module Gitlab
         if trans && proxy_start
           # Time in milliseconds since gitlab-workhorse started the request
           duration = Time.now.to_f * 1_000 - proxy_start.to_f / 1_000_000
-          trans.set(:rails_queue_duration, duration)
+          trans.set(:gitlab_rails_queue_duration_total, duration)
 
           duration_s = Gitlab::Utils.ms_to_round_sec(duration)
-          metric_rails_queue_duration_seconds.observe(trans.labels, duration_s)
+          trans.observe(:gitlab_rails_queue_duration_seconds, duration_s) do
+            docstring 'Measures latency between GitLab Workhorse forwarding a request to Rails'
+          end
           env[GITLAB_RAILS_QUEUE_DURATION_KEY] = duration_s
         end
 
         @app.call(env)
       end
-
-      private
-
-      def metric_rails_queue_duration_seconds
-        @metric_rails_queue_duration_seconds ||= Gitlab::Metrics.histogram(
-          :gitlab_rails_queue_duration_seconds,
-          'Measures latency between GitLab Workhorse forwarding a request to Rails',
-          Gitlab::Metrics::Transaction::BASE_LABELS
-        )
-      end
     end
   end
 end

spec/lib/gitlab/diff/highlight_cache_spec.rb

@@ -214,16 +214,17 @@ RSpec.describe Gitlab::Diff::HighlightCache, :clean_gitlab_redis_cache do
   end
 
   describe 'metrics' do
-    it 'defines :gitlab_redis_diff_caching_memory_usage_bytes histogram' do
-      expect(described_class).to respond_to(:gitlab_redis_diff_caching_memory_usage_bytes)
-    end
+    let(:transaction) { Gitlab::Metrics::WebTransaction.new({} ) }
 
-    it 'defines :gitlab_redis_diff_caching_hit' do
-      expect(described_class).to respond_to(:gitlab_redis_diff_caching_hit)
+    before do
+      allow(cache).to receive(:current_transaction).and_return(transaction)
     end
 
-    it 'defines :gitlab_redis_diff_caching_miss' do
-      expect(described_class).to respond_to(:gitlab_redis_diff_caching_miss)
+    it 'observes :gitlab_redis_diff_caching_memory_usage_bytes' do
+      expect(transaction)
+        .to receive(:observe).with(:gitlab_redis_diff_caching_memory_usage_bytes, a_kind_of(Numeric))
+
+      cache.write_if_empty
     end
   end
 end

spec/lib/gitlab/git/blob_spec.rb

@@ -10,34 +10,64 @@ RSpec.describe Gitlab::Git::Blob, :seed_helper do
 
   describe 'initialize' do
     let(:blob) { Gitlab::Git::Blob.new(name: 'test') }
+    let(:transaction) { Gitlab::Metrics::WebTransaction.new( {} ) }
+
+    before do
+      allow(::Gitlab::Metrics::Transaction).to receive(:current).and_return(transaction)
+    end
 
     it 'handles nil data' do
-      expect(described_class).not_to receive(:gitlab_blob_size)
+      expect(blob).not_to receive(:record_metric_blob_size)
 
       expect(blob.name).to eq('test')
       expect(blob.size).to eq(nil)
       expect(blob.loaded_size).to eq(nil)
     end
 
-    it 'records blob size' do
-      expect(described_class).to receive(:gitlab_blob_size).and_call_original
+    context 'with size' do
+      let(:blob1) { Gitlab::Git::Blob.new(name: 'test', size: 4, data: 'abcd') }
+
+      it 'observes gitlab_blob_size' do
+        expect(transaction)
+          .to receive(:observe).with(:gitlab_blob_size, a_kind_of(Numeric))
 
-      Gitlab::Git::Blob.new(name: 'test', size: 4, data: 'abcd')
+        blob1
+      end
     end
 
     context 'when untruncated' do
-      it 'attempts to record gitlab_blob_truncated_false' do
-        expect(described_class).to receive(:gitlab_blob_truncated_false).and_call_original
+      let(:blob) { Gitlab::Git::Blob.new(name: 'test', size: 4, data: 'abcd') }
+
+      it 'doesnt increment :gitlab_blob_truncated_true counter' do
+        expect(transaction)
+          .not_to receive(:increment).with(:gitlab_blob_truncated_true)
 
-        Gitlab::Git::Blob.new(name: 'test', size: 4, data: 'abcd')
+        blob
+      end
+
+      it 'increment :gitlab_blob_truncated_false counter' do
+        expect(transaction)
+          .to receive(:increment).with(:gitlab_blob_truncated_false)
+
+        blob
       end
     end
 
     context 'when truncated' do
-      it 'attempts to record gitlab_blob_truncated_true' do
-        expect(described_class).to receive(:gitlab_blob_truncated_true).and_call_original
+      let(:blob) { Gitlab::Git::Blob.new(name: 'test', size: 40, data: 'abcd') }
 
-        Gitlab::Git::Blob.new(name: 'test', size: 40, data: 'abcd')
+      it 'increment :gitlab_blob_truncated_true counter' do
+        expect(transaction)
+          .to receive(:increment).with(:gitlab_blob_truncated_true)
+
+        blob
+      end
+
+      it 'doesnt increment :gitlab_blob_truncated_false counter' do
+        expect(transaction)
+          .not_to receive(:increment).with(:gitlab_blob_truncated_false)
+
+        blob
       end
     end
   end
@@ -639,20 +669,6 @@ RSpec.describe Gitlab::Git::Blob, :seed_helper do
     end
   end
 
-  describe 'metrics' do
-    it 'defines :gitlab_blob_truncated_true counter' do
-      expect(described_class).to respond_to(:gitlab_blob_truncated_true)
-    end
-
-    it 'defines :gitlab_blob_truncated_false counter' do
-      expect(described_class).to respond_to(:gitlab_blob_truncated_false)
-    end
-
-    it 'defines :gitlab_blob_size histogram' do
-      expect(described_class).to respond_to(:gitlab_blob_size)
-    end
-  end
-
   describe '#lines' do
     context 'when the encoding cannot be detected' do
       it 'successfully splits the data' do

spec/lib/gitlab/metrics/elasticsearch_rack_middleware_spec.rb

@@ -9,8 +9,8 @@ RSpec.describe Gitlab::Metrics::ElasticsearchRackMiddleware do
   let(:transaction) { Gitlab::Metrics::WebTransaction.new(env) }
 
   describe '#call' do
-    let(:counter) { instance_double(Prometheus::Client::Counter, increment: nil) }
-    let(:histogram) { instance_double(Prometheus::Client::Histogram, observe: nil) }
+    # let(:counter) { instance_double(Prometheus::Client::Counter, increment: nil) }
+    # let(:histogram) { instance_double(Prometheus::Client::Histogram, observe: nil) }
     let(:elasticsearch_query_time) { 0.1 }
     let(:elasticsearch_requests_count) { 2 }
 
@@ -18,19 +18,6 @@ RSpec.describe Gitlab::Metrics::ElasticsearchRackMiddleware do
       allow(Gitlab::Instrumentation::ElasticsearchTransport).to receive(:query_time) { elasticsearch_query_time }
       allow(Gitlab::Instrumentation::ElasticsearchTransport).to receive(:get_request_count) { elasticsearch_requests_count }
 
-      allow(Gitlab::Metrics).to receive(:counter)
-        .with(:http_elasticsearch_requests_total,
-              an_instance_of(String),
-              Gitlab::Metrics::Transaction::BASE_LABELS)
-        .and_return(counter)
-
-      allow(Gitlab::Metrics).to receive(:histogram)
-        .with(:http_elasticsearch_requests_duration_seconds,
-              an_instance_of(String),
-              Gitlab::Metrics::Transaction::BASE_LABELS,
-              described_class::HISTOGRAM_BUCKETS)
-        .and_return(histogram)
-
       allow(Gitlab::Metrics).to receive(:current_transaction).and_return(transaction)
     end
 
@@ -39,15 +26,15 @@ RSpec.describe Gitlab::Metrics::ElasticsearchRackMiddleware do
     end
 
     it 'records elasticsearch metrics' do
-      expect(counter).to receive(:increment).with(transaction.labels, elasticsearch_requests_count)
-      expect(histogram).to receive(:observe).with(transaction.labels, elasticsearch_query_time)
+      expect(transaction).to receive(:increment).with(:http_elasticsearch_requests_total, elasticsearch_requests_count)
+      expect(transaction).to receive(:observe).with(:http_elasticsearch_requests_duration_seconds, elasticsearch_query_time)
 
       middleware.call(env)
     end
 
     it 'records elasticsearch metrics if an error is raised' do
-      expect(counter).to receive(:increment).with(transaction.labels, elasticsearch_requests_count)
-      expect(histogram).to receive(:observe).with(transaction.labels, elasticsearch_query_time)
+      expect(transaction).to receive(:increment).with(:http_elasticsearch_requests_total, elasticsearch_requests_count)
+      expect(transaction).to receive(:observe).with(:http_elasticsearch_requests_duration_seconds, elasticsearch_query_time)
 
       allow(app).to receive(:call).with(env).and_raise(StandardError)
 

spec/lib/gitlab/metrics/method_call_spec.rb

@@ -3,12 +3,12 @@
 require 'spec_helper'
 
 RSpec.describe Gitlab::Metrics::MethodCall do
-  let(:transaction) { double(:transaction, labels: {}) }
+  let(:transaction) { Gitlab::Metrics::WebTransaction.new({}) }
   let(:method_call) { described_class.new('Foo#bar', :Foo, '#bar', transaction) }
 
   describe '#measure' do
     after do
-      described_class.reload_metric!(:gitlab_method_call_duration_seconds)
+      ::Gitlab::Metrics::Transaction.reload_metric!(:gitlab_method_call_duration_seconds)
     end
 
     it 'measures the performance of the supplied block' do
@@ -36,13 +36,13 @@ RSpec.describe Gitlab::Metrics::MethodCall do
         end
 
         it 'metric is not a NullMetric' do
-          expect(described_class).not_to be_instance_of(Gitlab::Metrics::NullMetric)
+          method_call.measure { 'foo' }
+          expect(::Gitlab::Metrics::Transaction.prometheus_metric(:gitlab_method_call_duration_seconds, :histogram)).not_to be_instance_of(Gitlab::Metrics::NullMetric)
         end
 
         it 'observes the performance of the supplied block' do
-          expect(described_class.gitlab_method_call_duration_seconds)
-            .to receive(:observe)
-                  .with({ module: :Foo, method: '#bar' }, be_a_kind_of(Numeric))
+          expect(transaction)
+            .to receive(:observe).with(:gitlab_method_call_duration_seconds, be_a_kind_of(Numeric))
 
           method_call.measure { 'foo' }
         end
@@ -53,11 +53,17 @@ RSpec.describe Gitlab::Metrics::MethodCall do
           stub_feature_flags(prometheus_metrics_method_instrumentation: false)
         end
 
-        it 'observes using NullMetric' do
-          expect(described_class.gitlab_method_call_duration_seconds).to be_instance_of(Gitlab::Metrics::NullMetric)
-          expect(described_class.gitlab_method_call_duration_seconds).to receive(:observe)
+        it 'observes the performance of the supplied block' do
+          expect(transaction)
+            .to receive(:observe).with(:gitlab_method_call_duration_seconds, be_a_kind_of(Numeric))
+
+          method_call.measure { 'foo' }
+        end
 
+        it 'observes using NullMetric' do
           method_call.measure { 'foo' }
+
+          expect(::Gitlab::Metrics::Transaction.prometheus_metric(:gitlab_method_call_duration_seconds, :histogram)).to be_instance_of(Gitlab::Metrics::NullMetric)
         end
       end
     end
@@ -68,8 +74,9 @@ RSpec.describe Gitlab::Metrics::MethodCall do
       end
 
       it 'does not observe the performance' do
-        expect(described_class.gitlab_method_call_duration_seconds)
+        expect(transaction)
           .not_to receive(:observe)
+                .with(:gitlab_method_call_duration_seconds, be_a_kind_of(Numeric))
 
         method_call.measure { 'foo' }
       end

spec/lib/gitlab/metrics/methods_spec.rb

@@ -9,9 +9,9 @@ RSpec.describe Gitlab::Metrics::Methods do
     let(:docstring) { 'description' }
     let(:metric_name) { :sample_metric }
 
-    describe "#define_#{metric_type}" do
+    describe "#define_metrics" do
       define_method(:call_define_metric_method) do |**args|
-        subject.__send__("define_#{metric_type}", metric_name, **args)
+        subject.__send__(:define_metric, metric_type, metric_name, **args)
       end
 
       context 'metrics access method not defined' do
@@ -55,11 +55,11 @@ RSpec.describe Gitlab::Metrics::Methods do
       end
     end
 
-    describe "#fetch_#{metric_type}" do
+    describe "#fetch_metric" do
       let(:null_metric) { Gitlab::Metrics::NullMetric.instance }
 
       define_method(:call_fetch_metric_method) do |**args|
-        subject.__send__("fetch_#{metric_type}", metric_name, **args)
+        subject.__send__(:fetch_metric, metric_type, metric_name, **args)
       end
 
       context "when #{metric_type} is not cached" do

spec/lib/gitlab/metrics/rack_middleware_spec.rb

@@ -25,12 +25,4 @@ RSpec.describe Gitlab::Metrics::RackMiddleware do
       expect { middleware.call(env) }.to raise_error(RuntimeError)
     end
   end
-
-  describe '#transaction_from_env' do
-    let(:transaction) { middleware.transaction_from_env(env) }
-
-    it 'returns a Transaction' do
-      expect(transaction).to be_an_instance_of(Gitlab::Metrics::WebTransaction)
-    end
-  end
 end

spec/lib/gitlab/metrics/redis_rack_middleware_spec.rb

@@ -13,28 +13,12 @@ RSpec.describe Gitlab::Metrics::RedisRackMiddleware do
   end
 
   describe '#call' do
-    let(:counter) { double(Prometheus::Client::Counter, increment: nil) }
-    let(:histogram) { double(Prometheus::Client::Histogram, observe: nil) }
     let(:redis_query_time) { 0.1 }
     let(:redis_requests_count) { 2 }
 
     before do
       allow(Gitlab::Instrumentation::Redis).to receive(:query_time) { redis_query_time }
       allow(Gitlab::Instrumentation::Redis).to receive(:get_request_count) { redis_requests_count }
-
-      allow(Gitlab::Metrics).to receive(:counter)
-        .with(:http_redis_requests_total,
-              an_instance_of(String),
-              Gitlab::Metrics::Transaction::BASE_LABELS)
-        .and_return(counter)
-
-      allow(Gitlab::Metrics).to receive(:histogram)
-        .with(:http_redis_requests_duration_seconds,
-              an_instance_of(String),
-              Gitlab::Metrics::Transaction::BASE_LABELS,
-              Gitlab::Instrumentation::Redis::QUERY_TIME_BUCKETS)
-        .and_return(histogram)
-
       allow(Gitlab::Metrics).to receive(:current_transaction).and_return(transaction)
     end
 
@@ -43,15 +27,15 @@ RSpec.describe Gitlab::Metrics::RedisRackMiddleware do
     end
 
     it 'records redis metrics' do
-      expect(counter).to receive(:increment).with(transaction.labels, redis_requests_count)
-      expect(histogram).to receive(:observe).with(transaction.labels, redis_query_time)
+      expect(transaction).to receive(:increment).with(:http_redis_requests_total, redis_requests_count)
+      expect(transaction).to receive(:observe).with(:http_redis_requests_duration_seconds, redis_query_time)
 
       middleware.call(env)
     end
 
     it 'records redis metrics if an error is raised' do
-      expect(counter).to receive(:increment).with(transaction.labels, redis_requests_count)
-      expect(histogram).to receive(:observe).with(transaction.labels, redis_query_time)
+      expect(transaction).to receive(:increment).with(:http_redis_requests_total, redis_requests_count)
+      expect(transaction).to receive(:observe).with(:http_redis_requests_duration_seconds, redis_query_time)
 
       allow(app).to receive(:call).with(env).and_raise(StandardError)
 

spec/lib/gitlab/metrics/subscribers/action_view_spec.rb

@@ -22,15 +22,15 @@ RSpec.describe Gitlab::Metrics::Subscribers::ActionView do
   describe '#render_template' do
     it 'tracks rendering of a template' do
       expect(transaction).to receive(:increment)
-        .with(:view_duration, 2.1)
+        .with(:gitlab_transaction_view_duration_total, 2.1)
 
       subscriber.render_template(event)
     end
 
     it 'observes view rendering time' do
-      expect(described_class.gitlab_view_rendering_duration_seconds)
+      expect(transaction)
         .to receive(:observe)
-              .with({ view: 'app/views/x.html.haml' }, 2.1)
+        .with(:gitlab_view_rendering_duration_seconds, 2.1)
 
       subscriber.render_template(event)
     end

spec/lib/gitlab/metrics/subscribers/rails_cache_spec.rb

@@ -26,21 +26,12 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do
       context 'with hit event' do
         let(:event) { double(:event, duration: 15.2, payload: { hit: true }) }
 
-        it 'increments the cache_read_hit count' do
-          expect(transaction).to receive(:increment)
-                                   .with(:cache_read_hit_count, 1, false)
-          expect(transaction).to receive(:increment)
-                                   .with(any_args).at_least(1) # Other calls
-
-          subscriber.cache_read(event)
-        end
-
         context 'when super operation is fetch' do
           let(:event) { double(:event, duration: 15.2, payload: { hit: true, super_operation: :fetch }) }
 
-          it 'does not increment cache read miss' do
+          it 'does not increment cache read miss total' do
             expect(transaction).not_to receive(:increment)
-                                         .with(:cache_read_hit_count, 1)
+                                         .with(:gitlab_cache_misses_total, 1)
 
             subscriber.cache_read(event)
           end
@@ -50,33 +41,21 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do
       context 'with miss event' do
         let(:event) { double(:event, duration: 15.2, payload: { hit: false }) }
 
-        it 'increments the cache_read_miss count' do
+        it 'increments the cache_read_miss total' do
           expect(transaction).to receive(:increment)
-                                   .with(:cache_read_miss_count, 1, false)
+                                   .with(:gitlab_cache_misses_total, 1)
           expect(transaction).to receive(:increment)
                                    .with(any_args).at_least(1) # Other calls
 
           subscriber.cache_read(event)
         end
 
-        it 'increments the cache_read_miss total' do
-          expect(subscriber.send(:metric_cache_misses_total)).to receive(:increment).with({})
-
-          subscriber.cache_read(event)
-        end
-
         context 'when super operation is fetch' do
           let(:event) { double(:event, duration: 15.2, payload: { hit: false, super_operation: :fetch }) }
 
-          it 'does not increment cache read miss' do
+          it 'does not increment cache read miss total' do
             expect(transaction).not_to receive(:increment)
-                                         .with(:cache_read_miss_count, 1)
-
-            subscriber.cache_read(event)
-          end
-
-          it 'does not increment cache_read_miss total' do
-            expect(subscriber.send(:metric_cache_misses_total)).not_to receive(:increment).with({})
+                                         .with(:gitlab_cache_misses_total, 1)
 
             subscriber.cache_read(event)
           end
@@ -129,7 +108,7 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do
 
       it 'increments the cache_read_hit count' do
         expect(transaction).to receive(:increment)
-                                 .with(:cache_read_hit_count, 1)
+                                 .with(:gitlab_transaction_cache_read_hit_count_total, 1)
 
         subscriber.cache_fetch_hit(event)
       end
@@ -146,25 +125,17 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do
     end
 
     context 'with a transaction' do
-      let(:metric_cache_misses_total) { double('metric_cache_misses_total', increment: nil) }
-
       before do
-        allow(subscriber).to receive(:metric_cache_misses_total).and_return(metric_cache_misses_total)
         allow(subscriber).to receive(:current_transaction)
                                .and_return(transaction)
       end
 
-      it 'increments the cache_fetch_miss count' do
+      it 'increments the cache_fetch_miss count and cache_read_miss total' do
+        expect(transaction).to receive(:increment).with(:gitlab_cache_misses_total, 1)
         expect(transaction).to receive(:increment)
-                                 .with(:cache_read_miss_count, 1)
-
-        subscriber.cache_generate(event)
-      end
+                                 .with(:gitlab_transaction_cache_read_miss_count_total, 1)
 
-      it 'increments the cache_read_miss total' do
         subscriber.cache_generate(event)
-
-        expect(metric_cache_misses_total).to have_received(:increment).with({})
       end
     end
   end
@@ -184,34 +155,18 @@ RSpec.describe Gitlab::Metrics::Subscribers::RailsCache do
                                .and_return(transaction)
       end
 
-      it 'increments the total and specific cache duration' do
-        expect(transaction).to receive(:increment)
-                                 .with(:cache_duration, event.duration, false)
-
-        expect(transaction).to receive(:increment)
-                                 .with(:cache_count, 1, false)
-
-        expect(transaction).to receive(:increment)
-                                 .with(:cache_delete_duration, event.duration, false)
-
-        expect(transaction).to receive(:increment)
-                                 .with(:cache_delete_count, 1, false)
-
-        subscriber.observe(:delete, event.duration)
-      end
-
       it 'observes cache metric' do
-        expect(subscriber.send(:metric_cache_operation_duration_seconds))
+        expect(transaction)
           .to receive(:observe)
-          .with({ operation: :delete }, event.duration / 1000.0)
+          .with(:gitlab_cache_operation_duration_seconds, event.duration / 1000.0)
 
         subscriber.observe(:delete, event.duration)
       end
 
       it 'increments the operations total' do
-        expect(subscriber.send(:metric_cache_operations_total))
+        expect(transaction)
           .to receive(:increment)
-          .with(transaction.labels.merge(operation: :delete))
+          .with(:gitlab_cache_operations_total, 1)
 
         subscriber.observe(:delete, event.duration)
       end

spec/lib/gitlab/metrics/transaction_spec.rb

@@ -63,7 +63,7 @@ RSpec.describe Gitlab::Metrics::Transaction do
   end
 
   describe '#add_event' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil) }
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil, base_labels: {}) }
 
     it 'adds a metric' do
       expect(prometheus_metric).to receive(:increment)
@@ -82,7 +82,7 @@ RSpec.describe Gitlab::Metrics::Transaction do
     context 'with sensitive tags' do
       before do
         transaction.add_event(:baubau, **sensitive_tags.merge(sane: 'yes'))
-        allow(described_class).to receive(:transaction_metric).and_return(prometheus_metric)
+        allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric)
       end
 
       it 'filters tags' do
@@ -94,24 +94,119 @@ RSpec.describe Gitlab::Metrics::Transaction do
   end
 
   describe '#increment' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil) }
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, increment: nil, base_labels: {}) }
 
     it 'adds a metric' do
-      expect(prometheus_metric).to receive(:increment).with(hash_including(:action, :controller), 1)
-      expect(described_class).to receive(:fetch_metric).with(:counter, :gitlab_transaction_meow_total).and_return(prometheus_metric)
+      expect(prometheus_metric).to receive(:increment)
+      expect(::Gitlab::Metrics).to receive(:counter).with(:meow, 'Meow counter', hash_including(:controller, :action)).and_return(prometheus_metric)
 
       transaction.increment(:meow, 1)
     end
+
+    context 'with block' do
+      it 'overrides docstring' do
+        expect(::Gitlab::Metrics).to receive(:counter).with(:block_docstring, 'test', hash_including(:controller, :action)).and_return(prometheus_metric)
+
+        transaction.increment(:block_docstring, 1) do
+          docstring 'test'
+        end
+      end
+
+      it 'overrides labels' do
+        expect(::Gitlab::Metrics).to receive(:counter).with(:block_labels, 'Block labels counter', hash_including(:controller, :action, :sane)).and_return(prometheus_metric)
+
+        labels = { sane: 'yes' }
+        transaction.increment(:block_labels, 1) do
+          base_labels labels
+        end
+      end
+
+      it 'filters sensitive tags' do
+        expect(::Gitlab::Metrics).to receive(:counter).with(:metric_with_sensitive_block, 'Metric with sensitive block counter', hash_excluding(sensitive_tags)).and_return(prometheus_metric)
+
+        labels = sensitive_tags
+        transaction.increment(:metric_with_sensitive_block, 1) do
+          base_labels labels
+        end
+      end
+    end
   end
 
   describe '#set' do
-    let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, set: nil) }
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Gauge, set: nil, base_labels: {}) }
 
     it 'adds a metric' do
-      expect(prometheus_metric).to receive(:set).with(hash_including(:action, :controller), 1)
-      expect(described_class).to receive(:fetch_metric).with(:gauge, :gitlab_transaction_meow_total).and_return(prometheus_metric)
+      expect(prometheus_metric).to receive(:set)
+      expect(::Gitlab::Metrics).to receive(:gauge).with(:meow_set, 'Meow set gauge', hash_including(:controller, :action), :livesum).and_return(prometheus_metric)
+
+      transaction.set(:meow_set, 1)
+    end
+
+    context 'with block' do
+      it 'overrides docstring' do
+        expect(::Gitlab::Metrics).to receive(:gauge).with(:block_docstring_set, 'test', hash_including(:controller, :action), :livesum).and_return(prometheus_metric)
+
+        transaction.set(:block_docstring_set, 1) do
+          docstring 'test'
+        end
+      end
+
+      it 'overrides labels' do
+        expect(::Gitlab::Metrics).to receive(:gauge).with(:block_labels_set, 'Block labels set gauge', hash_including(:controller, :action, :sane), :livesum).and_return(prometheus_metric)
+
+        labels = { sane: 'yes' }
+        transaction.set(:block_labels_set, 1) do
+          base_labels labels
+        end
+      end
 
-      transaction.set(:meow, 1)
+      it 'filters sensitive tags' do
+        expect(::Gitlab::Metrics).to receive(:gauge).with(:metric_set_with_sensitive_block, 'Metric set with sensitive block gauge', hash_excluding(sensitive_tags), :livesum).and_return(prometheus_metric)
+
+        labels = sensitive_tags
+        transaction.set(:metric_set_with_sensitive_block, 1) do
+          base_labels labels
+        end
+      end
+    end
+  end
+
+  describe '#observe' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Histogram, observe: nil, base_labels: {}) }
+
+    it 'adds a metric' do
+      expect(prometheus_metric).to receive(:observe)
+      expect(::Gitlab::Metrics).to receive(:histogram).with(:meow_observe, 'Meow observe histogram', hash_including(:controller, :action), kind_of(Array)).and_return(prometheus_metric)
+
+      transaction.observe(:meow_observe, 1)
+    end
+
+    context 'with block' do
+      it 'overrides docstring' do
+        expect(::Gitlab::Metrics).to receive(:histogram).with(:block_docstring_observe, 'test', hash_including(:controller, :action), kind_of(Array)).and_return(prometheus_metric)
+
+        transaction.observe(:block_docstring_observe, 1) do
+          docstring 'test'
+        end
+      end
+
+      it 'overrides labels' do
+        expect(::Gitlab::Metrics).to receive(:histogram).with(:block_labels_observe, 'Block labels observe histogram', hash_including(:controller, :action, :sane), kind_of(Array)).and_return(prometheus_metric)
+
+        labels = { sane: 'yes' }
+        transaction.observe(:block_labels_observe, 1) do
+          base_labels labels
+        end
+      end
+
+      it 'filters sensitive tags' do
+        expect(::Gitlab::Metrics).to receive(:histogram).with(:metric_observe_with_sensitive_block, 'Metric observe with sensitive block histogram', hash_excluding(sensitive_tags), kind_of(Array)).and_return(prometheus_metric)
+
+        labels = sensitive_tags
+        transaction.observe(:metric_observe_with_sensitive_block, 1) do
+          base_labels labels
+        end
+      end
     end
   end
 end

spec/lib/gitlab/metrics/web_transaction_spec.rb

@@ -5,10 +5,11 @@ require 'spec_helper'
 RSpec.describe Gitlab::Metrics::WebTransaction do
   let(:env) { {} }
   let(:transaction) { described_class.new(env) }
-  let(:prometheus_metric) { double("prometheus metric") }
+  let(:prometheus_metric) { instance_double(Prometheus::Client::Metric, base_labels: {}) }
 
   before do
-    allow(described_class).to receive(:transaction_metric).and_return(prometheus_metric)
+    allow(described_class).to receive(:prometheus_metric).and_return(prometheus_metric)
+    allow(transaction).to receive(:observe)
   end
 
   describe '#duration' do
@@ -53,22 +54,6 @@ RSpec.describe Gitlab::Metrics::WebTransaction do
     end
   end
 
-  describe '#increment' do
-    it 'increments a counter' do
-      expect(prometheus_metric).to receive(:increment).with({}, 1)
-
-      transaction.increment(:time, 1)
-    end
-  end
-
-  describe '#set' do
-    it 'sets a value' do
-      expect(prometheus_metric).to receive(:set).with({}, 10)
-
-      transaction.set(:number, 10)
-    end
-  end
-
   describe '#labels' do
     let(:request) { double(:request, format: double(:format, ref: :html)) }
     let(:controller_class) { double(:controller_class, name: 'TestController') }
@@ -144,6 +129,8 @@ RSpec.describe Gitlab::Metrics::WebTransaction do
   end
 
   describe '#add_event' do
+    let(:prometheus_metric) { instance_double(Prometheus::Client::Counter, :increment, base_labels: {}) }
+
     it 'adds a metric' do
       expect(prometheus_metric).to receive(:increment)
 

spec/lib/gitlab/metrics_spec.rb

@@ -71,14 +71,9 @@ RSpec.describe Gitlab::Metrics do
       end
 
       it 'adds a metric to the current transaction' do
-        expect(transaction).to receive(:increment)
-                                 .with('foo_real_time', a_kind_of(Numeric), false)
+        expect(transaction).to receive(:observe).with(:gitlab_foo_real_duration_seconds, a_kind_of(Numeric))
 
-        expect(transaction).to receive(:increment)
-                                 .with('foo_cpu_time', a_kind_of(Numeric), false)
-
-        expect(transaction).to receive(:increment)
-                                 .with('foo_call_count', 1, false)
+        expect(transaction).to receive(:observe).with(:gitlab_foo_cpu_duration_seconds, a_kind_of(Numeric))
 
         described_class.measure(:foo) { 10 }
       end

spec/lib/gitlab/middleware/rails_queue_duration_spec.rb

@@ -29,26 +29,19 @@ RSpec.describe Gitlab::Middleware::RailsQueueDuration do
 
       it 'sets proxy_flight_time and calls the app when the header is present' do
         env['HTTP_GITLAB_WORKHORSE_PROXY_START'] = '123'
-        expect(transaction).to receive(:set).with(:rails_queue_duration, an_instance_of(Float))
+        expect(transaction).to receive(:set).with(:gitlab_rails_queue_duration_total, an_instance_of(Float))
         expect(middleware.call(env)).to eq('yay')
       end
 
       it 'observes rails queue duration metrics and calls the app when the header is present' do
         env['HTTP_GITLAB_WORKHORSE_PROXY_START'] = '2000000000'
 
-        expect(middleware.send(:metric_rails_queue_duration_seconds)).to receive(:observe).with(transaction.labels, 1)
+        expect(transaction).to receive(:observe).with(:gitlab_rails_queue_duration_seconds, 1)
 
         Timecop.freeze(Time.at(3)) do
           expect(middleware.call(env)).to eq('yay')
         end
       end
-
-      it 'creates a metric with a docstring' do
-        metric = middleware.send(:metric_rails_queue_duration_seconds)
-
-        expect(metric).to be_instance_of(Prometheus::Client::Histogram)
-        expect(metric.docstring).to eq('Measures latency between GitLab Workhorse forwarding a request to Rails')
-      end
     end
   end
 end