Add a separate histogram for builds queue size

This also renames labels for queue depth histogram to make them
connected to the returned result.

app/services/ci/register_job_service.rb

@@ -4,7 +4,7 @@ module Ci
   # This class responsible for assigning
   # proper pending build to runner on runner API request
   class RegisterJobService
-    attr_reader :runner
+    attr_reader :runner, :metrics
 
     Result = Struct.new(:build, :build_json, :valid?)
 
@@ -47,7 +47,7 @@ module Ci
         builds = builds.queued_before(params[:job_age].seconds.ago)
       end
 
-      @metrics.observe_queue_depth(:initialized, -> { builds.to_a.size })
+      @metrics.observe_queue_size(-> { builds.to_a.size })
 
       valid = true
       depth = 0
@@ -61,7 +61,7 @@ module Ci
 
         if result.valid?
           @metrics.register_success(result.build)
-          @metrics.observe_queue_depth(:effective, -> { depth })
+          @metrics.observe_queue_depth(:found, depth)
 
           return result # rubocop:disable Cop/AvoidReturnFromBlocks
         else
@@ -72,7 +72,8 @@ module Ci
       end
 
       @metrics.increment_queue_operation(:queue_conflict) unless valid
-      @metrics.observe_queue_depth(:ineffective, -> { depth }) unless valid
+      @metrics.observe_queue_depth(:conflict, depth) unless valid
+      @metrics.observe_queue_depth(:not_found, depth) if valid
       @metrics.register_failure
 
       Result.new(nil, nil, valid)

ee/app/services/ee/ci/register_job_service.rb

@@ -21,7 +21,8 @@ module EE
           # "Hi, we don't have any more builds now,  but not everything is right anyway, so try again".
           # Runner will retry, but again, against replica, and again will check if replication lag did catch-up.
           if !db_all_caught_up && !result.build
-            # TODO replication lag metric
+            metrics.increment_queue_operation(:queue_replication_lag)
+
             return ::Ci::RegisterJobService::Result.new(nil, false) # rubocop:disable Cop/AvoidReturnFromBlocks
           end
         end

lib/gitlab/ci/queue/metrics.rb

@@ -7,7 +7,8 @@ module Gitlab
         extend Gitlab::Utils::StrongMemoize
 
         QUEUE_DURATION_SECONDS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
-        QUEUE_DEPTH_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
+        QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
+        QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
         QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
 
         METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
@@ -23,14 +24,15 @@ module Gitlab
           :queue_attempt,
           :queue_conflict,
           :queue_iteration,
+          :queue_replication_lag,
           :runner_pre_assign_checks_failed,
           :runner_pre_assign_checks_success
         ].to_set.freeze
 
         QUEUE_DEPTH_HISTOGRAMS = [
-          :initialized,
-          :effective,
-          :ineffective
+          :found,
+          :not_found,
+          :conflict
         ].to_set.freeze
 
         attr_reader :runner
@@ -79,14 +81,20 @@ module Gitlab
           self.class.queue_operations_total.increment(operation: operation)
         end
 
-        def observe_queue_depth(queue, size_proc)
+        def observe_queue_depth(queue, size)
           return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
 
           if !Rails.env.production? && !QUEUE_DEPTH_HISTOGRAMS.include?(queue)
             raise ArgumentError, "unknown queue depth label: #{queue}"
           end
 
-          self.class.queue_depth_size_total.observe({ queue: queue }, size_proc.call.to_f)
+          self.class.queue_depth_total.observe({ queue: queue }, size.to_f)
+        end
+
+        def observe_queue_size(size_proc)
+          return unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
+
+          self.class.queue_size_total.observe({}, size_proc.call.to_f)
         end
 
         def observe_queue_time
@@ -140,12 +148,23 @@ module Gitlab
           end
         end
 
-        def self.queue_depth_size_total
-          strong_memoize(:queue_depth_size_total) do
-            name = :gitlab_ci_queue_depth_size_total
-            comment = 'Size of a CI/CD builds queue'
+        def self.queue_depth_total
+          strong_memoize(:queue_depth_total) do
+            name = :gitlab_ci_queue_depth_total
+            comment = 'Size of a CI/CD builds queue in relation to the operation result'
+            labels = {}
+            buckets = QUEUE_DEPTH_TOTAL_BUCKETS
+
+            Gitlab::Metrics.histogram(name, comment, labels, buckets)
+          end
+        end
+
+        def self.queue_size_total
+          strong_memoize(:queue_size_total) do
+            name = :gitlab_ci_queue_size_total
+            comment = 'Size of initialized CI/CD builds queue'
             labels = {}
-            buckets = QUEUE_DEPTH_SIZE_TOTAL_BUCKETS
+            buckets = QUEUE_SIZE_TOTAL_BUCKETS
 
             Gitlab::Metrics.histogram(name, comment, labels, buckets)
           end