Instrument builds queue SQL execution duration

This commit adds a Prometheus histogram used to observe the time it
takes to execute a SQL query we use to retrieve builds queue from the
database.

app/services/ci/register_job_service.rb

@@ -24,7 +24,7 @@ module Ci
     def execute(params = {})
       @metrics.increment_queue_operation(:queue_attempt)
 
-      @metrics.observe_queue_time do
+      @metrics.observe_queue_time(:process) do
         process_queue(params)
       end
     end
@@ -110,7 +110,7 @@ module Ci
       end
 
       if Feature.enabled?(:ci_register_job_service_one_by_one, runner, default_enabled: true)
-        build_ids = process_queue(-> { builds.pluck(:id) })
+        build_ids = retrieve_queue(-> { builds.pluck(:id) })
 
         @metrics.observe_queue_size(-> { build_ids.size })
 
@@ -118,7 +118,7 @@ module Ci
           yield Ci::Build.find(build_id)
         end
       else
-        builds = process_queue(-> { builds.to_a })
+        builds = retrieve_queue(-> { builds.to_a })
 
         @metrics.observe_queue_size(-> { builds.size })
 
@@ -127,9 +127,11 @@ module Ci
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
-    def process_queue(queue_query_proc)
+    def retrieve_queue(queue_query_proc)
+      @metrics.observe_queue_time(:retrieve) do
         queue_query_proc.call
       end
+    end
 
     def process_build(build, params)
       unless build.pending?

ee/app/models/ee/ci/runner.rb

@@ -9,8 +9,8 @@ module EE
         ##
         # We only stick a runner to primary database to be able to detect the
         # replication lag in `EE::Ci::RegisterJobService#execute`. The
-        # intention here is not execute `Ci::RegisterJobService#execute` on the
-        # primary database.
+        # intention here is not to execute `Ci::RegisterJobService#execute` on
+        # the primary database.
         #
         ::Gitlab::Database::LoadBalancing::Sticking.stick(:runner, id)
 

lib/gitlab/ci/queue/metrics.rb

@@ -10,7 +10,7 @@ module Gitlab
         QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
         QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
         QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
-        QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
+        QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
 
         METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
         DEFAULT_METRICS_SHARD = 'default'
@@ -100,7 +100,7 @@ module Gitlab
           self.class.queue_size_total.observe({}, size_proc.call.to_f)
         end
 
-        def observe_queue_time
+        def observe_queue_time(metric)
           start_time = ::Gitlab::Metrics::System.monotonic_time
 
           result = yield
@@ -108,7 +108,15 @@ module Gitlab
           return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
 
           seconds = ::Gitlab::Metrics::System.monotonic_time - start_time
+
+          case metric
+          when :process
             self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f)
+          when :retrieve
+            self.class.queue_retrieval_duration_seconds.observe({}, seconds.to_f)
+          else
+            raise ArgumentError unless Rails.env.production?
+          end
 
           result
         end
@@ -187,7 +195,18 @@ module Gitlab
           strong_memoize(:queue_iteration_duration_seconds) do
             name = :gitlab_ci_queue_iteration_duration_seconds
             comment = 'Time it takes to find a build in CI/CD queue'
-            buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS
+            buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
+            labels = {}
+
+            Gitlab::Metrics.histogram(name, comment, labels, buckets)
+          end
+        end
+
+        def self.queue_retrieval_duration_seconds
+          strong_memoize(:queue_retrival_duration_seconds) do
+            name = :gitlab_ci_queue_retrival_duration_seconds
+            comment = 'Time it takes to execute a SQL query to retrieve builds queue'
+            buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
             labels = {}
 
             Gitlab::Metrics.histogram(name, comment, labels, buckets)