Commit d4da1cc5 authored by Sean McGivern's avatar Sean McGivern

Merge branch 'feature/gb/builds-queue-on-replicas' into 'master'

Ensure that the builds queuing query is executed on replicas [RUN ALL RSPEC] [RUN AS-IF-FOSS]

See merge request gitlab-org/gitlab!56849
parents 650e992c 75c6b5e5
......@@ -24,7 +24,7 @@ module Ci
def execute(params = {})
@metrics.increment_queue_operation(:queue_attempt)
@metrics.observe_queue_time do
@metrics.observe_queue_time(:process) do
process_queue(params)
end
end
......@@ -110,7 +110,7 @@ module Ci
end
if Feature.enabled?(:ci_register_job_service_one_by_one, runner, default_enabled: true)
build_ids = builds.pluck(:id)
build_ids = retrieve_queue(-> { builds.pluck(:id) })
@metrics.observe_queue_size(-> { build_ids.size })
......@@ -118,13 +118,21 @@ module Ci
yield Ci::Build.find(build_id)
end
else
@metrics.observe_queue_size(-> { builds.to_a.size })
builds_array = retrieve_queue(-> { builds.to_a })
builds.each(&blk)
@metrics.observe_queue_size(-> { builds_array.size })
builds_array.each(&blk)
end
end
# rubocop: enable CodeReuse/ActiveRecord
def retrieve_queue(queue_query_proc)
@metrics.observe_queue_time(:retrieve) do
queue_query_proc.call
end
end
def process_build(build, params)
unless build.pending?
@metrics.increment_queue_operation(:build_not_pending)
......
---
name: ci_runner_builds_queue_on_replicas
introduced_by_url:
rollout_issue_url:
milestone: '13.10'
type: development
group: group::continuous integration
default_enabled: false
......@@ -6,11 +6,28 @@ module EE
extend ActiveSupport::Concern
def tick_runner_queue
##
# We only stick a runner to primary database to be able to detect the
# replication lag in `EE::Ci::RegisterJobService#execute`. The
# intention here is not to execute `Ci::RegisterJobService#execute` on
# the primary database.
#
::Gitlab::Database::LoadBalancing::Sticking.stick(:runner, id)
super
end
def heartbeat(values)
return super unless ::Feature.enabled?(:ci_runner_builds_queue_on_replicas, self, default_enabled: :yaml)
##
# We can safely ignore writes performed by a runner heartbeat. We do
# not want to upgrade database connection proxy to use the primary
# database after heartbeat write happens.
#
::Gitlab::Database::LoadBalancing::Session.without_sticky_writes { super }
end
def minutes_cost_factor(access_level)
return 0.0 unless instance_type?
......
......@@ -28,6 +28,18 @@ module EE
end
end
def retrieve_queue(queue_query_proc)
if ::Feature.enabled?(:ci_runner_builds_queue_on_replicas, runner, default_enabled: :yaml)
##
# We want to reset a load balancing session to discard the side
# effects of writes that could have happened prior to this moment.
#
::Gitlab::Database::LoadBalancing::Session.clear_session
end
super
end
def builds_for_shared_runner
return super unless shared_runner_build_limits_feature_enabled?
......
......@@ -19,9 +19,14 @@ module Gitlab
RequestStore.delete(CACHE_KEY)
end
def self.without_sticky_writes(&block)
current.ignore_writes(&block)
end
def initialize
@use_primary = false
@performed_write = false
@ignore_writes = false
end
def use_primary?
......@@ -42,8 +47,19 @@ module Gitlab
@use_primary = used_primary || @performed_write
end
def ignore_writes(&block)
@ignore_writes = true
yield
ensure
@ignore_writes = false
end
def write!
@performed_write = true
return if @ignore_writes
use_primary!
end
......
......@@ -22,6 +22,28 @@ RSpec.describe Gitlab::Database::LoadBalancing::Session do
end
end
describe '.without_sticky_writes' do
it 'ignores sticky write events sent by a connection proxy' do
described_class.without_sticky_writes do
described_class.current.write!
end
session = described_class.current
expect(session).not_to be_using_primary
end
it 'still is aware of write that happened' do
described_class.without_sticky_writes do
described_class.current.write!
end
session = described_class.current
expect(session.performed_write?).to be true
end
end
describe '#use_primary?' do
it 'returns true when the primary should be used' do
instance = described_class.new
......@@ -96,4 +118,24 @@ RSpec.describe Gitlab::Database::LoadBalancing::Session do
expect(instance.performed_write?).to eq(true)
end
end
describe '#ignore_writes' do
it 'ignores write events' do
instance = described_class.new
instance.ignore_writes { instance.write! }
expect(instance).not_to be_using_primary
expect(instance.performed_write?).to eq true
end
it 'does not prevent using primary if an exception is raised' do
instance = described_class.new
instance.ignore_writes { raise ArgumentError } rescue ArgumentError
instance.write!
expect(instance).to be_using_primary
end
end
end
......@@ -10,7 +10,7 @@ module Gitlab
QUEUE_ACTIVE_RUNNERS_BUCKETS = [1, 3, 10, 30, 60, 300, 900, 1800, 3600].freeze
QUEUE_DEPTH_TOTAL_BUCKETS = [1, 2, 3, 5, 8, 16, 32, 50, 100, 250, 500, 1000, 2000, 5000].freeze
QUEUE_SIZE_TOTAL_BUCKETS = [1, 5, 10, 50, 100, 500, 1000, 2000, 5000].freeze
QUEUE_ITERATION_DURATION_SECONDS_BUCKETS = [0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS = [0.01, 0.05, 0.1, 0.3, 0.5, 1, 5, 10, 30, 60, 180, 300].freeze
METRICS_SHARD_TAG_PREFIX = 'metrics_shard::'
DEFAULT_METRICS_SHARD = 'default'
......@@ -100,7 +100,7 @@ module Gitlab
self.class.queue_size_total.observe({}, size_proc.call.to_f)
end
def observe_queue_time
def observe_queue_time(metric)
start_time = ::Gitlab::Metrics::System.monotonic_time
result = yield
......@@ -108,7 +108,15 @@ module Gitlab
return result unless Feature.enabled?(:gitlab_ci_builds_queuing_metrics, default_enabled: false)
seconds = ::Gitlab::Metrics::System.monotonic_time - start_time
case metric
when :process
self.class.queue_iteration_duration_seconds.observe({}, seconds.to_f)
when :retrieve
self.class.queue_retrieval_duration_seconds.observe({}, seconds.to_f)
else
raise ArgumentError unless Rails.env.production?
end
result
end
......@@ -187,7 +195,18 @@ module Gitlab
strong_memoize(:queue_iteration_duration_seconds) do
name = :gitlab_ci_queue_iteration_duration_seconds
comment = 'Time it takes to find a build in CI/CD queue'
buckets = QUEUE_ITERATION_DURATION_SECONDS_BUCKETS
buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
labels = {}
Gitlab::Metrics.histogram(name, comment, labels, buckets)
end
end
def self.queue_retrieval_duration_seconds
strong_memoize(:queue_retrieval_duration_seconds) do
name = :gitlab_ci_queue_retrieval_duration_seconds
comment = 'Time it takes to execute a SQL query to retrieve builds queue'
buckets = QUEUE_PROCESSING_DURATION_SECONDS_BUCKETS
labels = {}
Gitlab::Metrics.histogram(name, comment, labels, buckets)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment