Commit ce4dd77e authored by Matthias Käppler's avatar Matthias Käppler Committed by Etienne Baqué

Consolidate life-cycle hooks in 7_prometheus

This also returns early unless we are executing
in a real application context.
parent dc8e4cf6
---
name: prometheus_initializer_refactor
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81133
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/353446
milestone: '14.9'
type: development
group: group::memory
default_enabled: true
# frozen_string_literal: true
return if Feature.feature_flags_available? && Feature.enabled?(:prometheus_initializer_refactor, default_enabled: :yaml)
# Keep separate directories for separate processes
def prometheus_default_multiproc_dir
return unless Rails.env.development? || Rails.env.test?
......@@ -58,7 +60,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1)
if Gitlab::Runtime.web_server?
if Gitlab::Runtime.puma?
Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics
end
......@@ -75,7 +77,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance(logger: logger).start
Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance(logger: logger).start
if Gitlab::Runtime.web_server?
if Gitlab::Runtime.puma?
Gitlab::Metrics::Samplers::ActionCableSampler.instance(logger: logger).start
end
......@@ -90,7 +92,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
end
end
if Gitlab::Runtime.web_server?
if Gitlab::Runtime.puma?
Gitlab::Cluster::LifecycleEvents.on_master_start do
Gitlab::Metrics::Exporter::WebExporter.instance.start
end
......
# frozen_string_literal: true
return unless Feature.feature_flags_available? && Feature.enabled?(:prometheus_initializer_refactor, default_enabled: :yaml)
# Keep separate directories for separate processes
def prometheus_default_multiproc_dir
return unless Rails.env.development? || Rails.env.test?
if Gitlab::Runtime.sidekiq?
Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
elsif Gitlab::Runtime.puma?
Rails.root.join('tmp/prometheus_multiproc_dir/puma')
else
Rails.root.join('tmp/prometheus_multiproc_dir')
end
end
::Prometheus::Client.configure do |config|
config.logger = Gitlab::AppLogger
config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] || prometheus_default_multiproc_dir
config.pid_provider = ::Prometheus::PidProvider.method(:worker_id)
end
Gitlab::Application.configure do |config|
# 0 should be Sentry to catch errors in this middleware
config.middleware.insert_after(Labkit::Middleware::Rack, Gitlab::Metrics::RequestsRackMiddleware)
end
# Any actions beyond this check should only execute outside of tests, when running in an application
# context (i.e. not in the Rails console or rspec) and when users have enabled metrics.
return if Rails.env.test? || !Gitlab::Runtime.application? || !Gitlab::Metrics.prometheus_metrics_enabled?
if Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER_ID'] == '0')
# The single worker outside of a sidekiq-cluster, or the first worker (sidekiq_0)
# in a cluster of processes, is responsible for serving health checks.
#
# Do not clean the metrics directory here - the supervisor script should
# have already taken care of that.
Sidekiq.configure_server do |config|
config.on(:startup) do
# In https://gitlab.com/gitlab-org/gitlab/-/issues/345804 we are looking to
# only serve health-checks from a worker process; for backwards compatibility
# we still go through the metrics exporter server, but start to configure it
# with the new settings keys.
exporter_settings = Settings.monitoring.sidekiq_health_checks
Gitlab::Metrics::Exporter::SidekiqExporter.instance(exporter_settings).start
end
end
end
Gitlab::Cluster::LifecycleEvents.on_master_start do
# When running Puma in a Single mode, `on_master_start` and `on_worker_start` are the same.
# Thus, we order these events to run `reinitialize_on_pid_change` with `force: true` first.
::Prometheus::Client.reinitialize_on_pid_change(force: true)
Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1)
if Gitlab::Runtime.puma?
Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics
Gitlab::Metrics::Samplers::PumaSampler.instance.start
# Starts a metrics server to export metrics from the Puma primary.
Gitlab::Metrics::Exporter::WebExporter.instance.start
end
Gitlab::Ci::Parsers.instrument!
rescue IOError => e
Gitlab::ErrorTracking.track_exception(e)
Gitlab::Metrics.error_detected!
end
Gitlab::Cluster::LifecycleEvents.on_worker_start do
defined?(::Prometheus::Client.reinitialize_on_pid_change) && ::Prometheus::Client.reinitialize_on_pid_change
logger = Gitlab::AppLogger
Gitlab::Metrics::Samplers::RubySampler.initialize_instance(logger: logger).start
Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance(logger: logger).start
Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance(logger: logger).start
if Gitlab::Runtime.puma?
# Since we are running a metrics server on the Puma primary, we would inherit
# this thread after forking into workers, so we need to explicitly stop it here.
# NOTE: This will not be necessary anymore after moving to an external server
# process via https://gitlab.com/gitlab-org/gitlab/-/issues/350548
Gitlab::Metrics::Exporter::WebExporter.instance.stop
Gitlab::Metrics::Samplers::ActionCableSampler.instance(logger: logger).start
end
if Gitlab.ee? && Gitlab::Runtime.sidekiq?
Gitlab::Metrics::Samplers::GlobalSearchSampler.instance(logger: logger).start
end
Gitlab::Ci::Parsers.instrument!
rescue IOError => e
Gitlab::ErrorTracking.track_exception(e)
Gitlab::Metrics.error_detected!
end
if Gitlab::Runtime.puma?
Gitlab::Cluster::LifecycleEvents.on_before_graceful_shutdown do
# We need to ensure that before we re-exec or shutdown server
# we do stop the exporter
Gitlab::Metrics::Exporter::WebExporter.instance.stop
end
Gitlab::Cluster::LifecycleEvents.on_before_master_restart do
# We need to ensure that before we re-exec server
# we do stop the exporter
#
# We do it again, for being extra safe,
# but it should not be needed
Gitlab::Metrics::Exporter::WebExporter.instance.stop
end
end
......@@ -383,7 +383,7 @@ What was done?
```ruby
# config/engines.rb
# Load only in case we are running web_server or rails console
if Gitlab::Runtime.web_server? || Gitlab::Runtime.console?
if Gitlab::Runtime.puma? || Gitlab::Runtime.console?
require 'web_engine'
end
```
......
......@@ -390,7 +390,7 @@ module Gitlab
end
def self.long_timeout
if Gitlab::Runtime.web_server?
if Gitlab::Runtime.puma?
default_timeout
else
6.hours
......
......@@ -16,7 +16,7 @@ module Gitlab
def disk_access_denied?
return true unless ::Settings.pages.local_store&.enabled
::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite?
::Gitlab::Runtime.puma? && !::Gitlab::Runtime.test_suite?
end
def report_denied_disk_access
......
......@@ -65,12 +65,15 @@ module Gitlab
!!defined?(::Rails::Command::RunnerCommand)
end
def web_server?
puma?
# Whether we are executing in an actual application context i.e. Puma or Sidekiq.
def application?
puma? || sidekiq?
end
# Whether we are executing in a multi-threaded environment. For now this is equivalent
# to meaning Puma or Sidekiq, but this could change in the future.
def multi_threaded?
puma? || sidekiq?
application?
end
def puma_in_clustered_mode?
......@@ -94,7 +97,7 @@ module Gitlab
threads += Sidekiq.options[:concurrency] + 2
end
if web_server?
if puma?
threads += Gitlab::ActionCable::Config.worker_pool_size
end
......
......@@ -13,7 +13,7 @@ RSpec.describe Gitlab::Pages::Settings do
context 'when running under a web server outside of test mode' do
before do
allow(::Gitlab::Runtime).to receive(:test_suite?).and_return(false)
allow(::Gitlab::Runtime).to receive(:web_server?).and_return(true)
allow(::Gitlab::Runtime).to receive(:puma?).and_return(true)
end
it 'logs a DiskAccessDenied error' do
......
......@@ -80,6 +80,10 @@ RSpec.describe Gitlab::Runtime do
it_behaves_like "valid runtime", :puma, 3 + Gitlab::ActionCable::Config.worker_pool_size
it 'identifies as an application runtime' do
expect(Gitlab::Runtime.application?).to be true
end
context "when ActionCable worker pool size is configured" do
before do
stub_env('ACTION_CABLE_WORKER_POOL_SIZE', 10)
......@@ -113,6 +117,10 @@ RSpec.describe Gitlab::Runtime do
end
it_behaves_like "valid runtime", :sidekiq, 5
it 'identifies as an application runtime' do
expect(Gitlab::Runtime.application?).to be true
end
end
context "console" do
......@@ -121,6 +129,10 @@ RSpec.describe Gitlab::Runtime do
end
it_behaves_like "valid runtime", :console, 1
it 'does not identify as an application runtime' do
expect(Gitlab::Runtime.application?).to be false
end
end
context "test suite" do
......@@ -129,6 +141,10 @@ RSpec.describe Gitlab::Runtime do
end
it_behaves_like "valid runtime", :test_suite, 1
it 'does not identify as an application runtime' do
expect(Gitlab::Runtime.application?).to be false
end
end
context "geo log cursor" do
......@@ -145,5 +161,9 @@ RSpec.describe Gitlab::Runtime do
end
it_behaves_like "valid runtime", :rails_runner, 1
it 'does not identify as an application runtime' do
expect(Gitlab::Runtime.application?).to be false
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment