Join with pending builds table to speed up queuing

This change makes it possible to perform a join with a separate table
that stores the queue of pending builds.

Instead of traversing a very large table that holds all builds that have
been processed and these that are pending too, we maintain a small table
that just stores a small amount of builds that are in the pending state.

app/models/ci/build.rb

@@ -1076,6 +1076,10 @@ module Ci
       ::Ci::PendingBuild.where(build_id: self.id)
     end
 
+    def create_queuing_entry!
+      ::Ci::PendingBuild.upsert_from_build!(self)
+    end
+
     protected
 
     def run_status_commit_hooks!

app/services/ci/register_job_service.rb

@@ -300,8 +300,16 @@ module Ci
     end
     # rubocop: enable CodeReuse/ActiveRecord
 
+    def all_builds
+      if Feature.enabled?(:ci_pending_builds_queue_join, runner, default_enabled: :yaml)
+        Ci::Build.joins(:queuing_entry)
+      else
+        Ci::Build.all
+      end
+    end
+
     def new_builds
-      builds = Ci::Build.pending.unstarted
+      builds = all_builds.pending.unstarted
       builds = builds.ref_protected if runner.ref_protected?
       builds
     end

app/services/ci/update_build_queue_service.rb

@@ -19,7 +19,7 @@ module Ci
       raise InvalidQueueTransition unless transition.to == 'pending'
 
       transition.within_transaction do
-        result = ::Ci::PendingBuild.upsert_from_build!(build)
+        result = build.create_queuing_entry!
 
         unless result.empty?
           metrics.increment_queue_operation(:build_queue_push)

config/feature_flags/development/ci_pending_builds_queue_join.yml

+---
+name: ci_pending_builds_queue_join
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/62195
+rollout_issue_url:
+milestone: '13.12'
+type: development
+group: group::continuous integration
+default_enabled: false

spec/factories/ci/builds.rb

@@ -78,10 +78,17 @@ FactoryBot.define do
     end
 
     trait :pending do
+      queued
       queued_at { 'Di 29. Okt 09:50:59 CET 2013' }
       status { 'pending' }
     end
 
+    trait :queued do
+      after(:create) do |build|
+        build.create_queuing_entry!
+      end
+    end
+
     trait :created do
       status { 'created' }
     end