Merge branch '350548-clean-up-prometheus-initializer' into 'master'

Clean up 7_prometheus initializer

See merge request gitlab-org/gitlab!81133

config/feature_flags/development/prometheus_initializer_refactor.yml

+---
+name: prometheus_initializer_refactor
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/81133
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/353446
+milestone: '14.9'
+type: development
+group: group::memory
+default_enabled: true

config/initializers/7_prometheus_metrics.rb

 # frozen_string_literal: true
 
+return if Feature.feature_flags_available? && Feature.enabled?(:prometheus_initializer_refactor, default_enabled: :yaml)
+
 # Keep separate directories for separate processes
 def prometheus_default_multiproc_dir
   return unless Rails.env.development? || Rails.env.test?
@@ -58,7 +60,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
 
     Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1)
 
-    if Gitlab::Runtime.web_server?
+    if Gitlab::Runtime.puma?
       Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics
     end
 
@@ -75,7 +77,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
     Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance(logger: logger).start
     Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance(logger: logger).start
 
-    if Gitlab::Runtime.web_server?
+    if Gitlab::Runtime.puma?
       Gitlab::Metrics::Samplers::ActionCableSampler.instance(logger: logger).start
     end
 
@@ -90,7 +92,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
   end
 end
 
-if Gitlab::Runtime.web_server?
+if Gitlab::Runtime.puma?
   Gitlab::Cluster::LifecycleEvents.on_master_start do
     Gitlab::Metrics::Exporter::WebExporter.instance.start
   end

config/initializers/7_prometheus_metrics_new.rb

+# frozen_string_literal: true
+
+return unless Feature.feature_flags_available? && Feature.enabled?(:prometheus_initializer_refactor, default_enabled: :yaml)
+
+# Keep separate directories for separate processes
+def prometheus_default_multiproc_dir
+  return unless Rails.env.development? || Rails.env.test?
+
+  if Gitlab::Runtime.sidekiq?
+    Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
+  elsif Gitlab::Runtime.puma?
+    Rails.root.join('tmp/prometheus_multiproc_dir/puma')
+  else
+    Rails.root.join('tmp/prometheus_multiproc_dir')
+  end
+end
+
+::Prometheus::Client.configure do |config|
+  config.logger = Gitlab::AppLogger
+
+  config.multiprocess_files_dir = ENV['prometheus_multiproc_dir'] || prometheus_default_multiproc_dir
+
+  config.pid_provider = ::Prometheus::PidProvider.method(:worker_id)
+end
+
+Gitlab::Application.configure do |config|
+  # 0 should be Sentry to catch errors in this middleware
+  config.middleware.insert_after(Labkit::Middleware::Rack, Gitlab::Metrics::RequestsRackMiddleware)
+end
+
+# Any actions beyond this check should only execute outside of tests, when running in an application
+# context (i.e. not in the Rails console or rspec) and when users have enabled metrics.
+return if Rails.env.test? || !Gitlab::Runtime.application? || !Gitlab::Metrics.prometheus_metrics_enabled?
+
+if Gitlab::Runtime.sidekiq? && (!ENV['SIDEKIQ_WORKER_ID'] || ENV['SIDEKIQ_WORKER_ID'] == '0')
+  # The single worker outside of a sidekiq-cluster, or the first worker (sidekiq_0)
+  # in a cluster of processes, is responsible for serving health checks.
+  #
+  # Do not clean the metrics directory here - the supervisor script should
+  # have already taken care of that.
+  Sidekiq.configure_server do |config|
+    config.on(:startup) do
+      # In https://gitlab.com/gitlab-org/gitlab/-/issues/345804 we are looking to
+      # only serve health-checks from a worker process; for backwards compatibility
+      # we still go through the metrics exporter server, but start to configure it
+      # with the new settings keys.
+      exporter_settings = Settings.monitoring.sidekiq_health_checks
+      Gitlab::Metrics::Exporter::SidekiqExporter.instance(exporter_settings).start
+    end
+  end
+end
+
+Gitlab::Cluster::LifecycleEvents.on_master_start do
+  # When running Puma in a Single mode, `on_master_start` and `on_worker_start` are the same.
+  # Thus, we order these events to run `reinitialize_on_pid_change` with `force: true` first.
+  ::Prometheus::Client.reinitialize_on_pid_change(force: true)
+
+  Gitlab::Metrics.gauge(:deployments, 'GitLab Version', {}, :max).set({ version: Gitlab::VERSION, revision: Gitlab.revision }, 1)
+
+  if Gitlab::Runtime.puma?
+    Gitlab::Metrics::RequestsRackMiddleware.initialize_metrics
+
+    Gitlab::Metrics::Samplers::PumaSampler.instance.start
+
+    # Starts a metrics server to export metrics from the Puma primary.
+    Gitlab::Metrics::Exporter::WebExporter.instance.start
+  end
+
+  Gitlab::Ci::Parsers.instrument!
+rescue IOError => e
+  Gitlab::ErrorTracking.track_exception(e)
+  Gitlab::Metrics.error_detected!
+end
+
+Gitlab::Cluster::LifecycleEvents.on_worker_start do
+  defined?(::Prometheus::Client.reinitialize_on_pid_change) && ::Prometheus::Client.reinitialize_on_pid_change
+  logger = Gitlab::AppLogger
+  Gitlab::Metrics::Samplers::RubySampler.initialize_instance(logger: logger).start
+  Gitlab::Metrics::Samplers::DatabaseSampler.initialize_instance(logger: logger).start
+  Gitlab::Metrics::Samplers::ThreadsSampler.initialize_instance(logger: logger).start
+
+  if Gitlab::Runtime.puma?
+    # Since we are running a metrics server on the Puma primary, we would inherit
+    # this thread after forking into workers, so we need to explicitly stop it here.
+    # NOTE: This will not be necessary anymore after moving to an external server
+    # process via https://gitlab.com/gitlab-org/gitlab/-/issues/350548
+    Gitlab::Metrics::Exporter::WebExporter.instance.stop
+
+    Gitlab::Metrics::Samplers::ActionCableSampler.instance(logger: logger).start
+  end
+
+  if Gitlab.ee? && Gitlab::Runtime.sidekiq?
+    Gitlab::Metrics::Samplers::GlobalSearchSampler.instance(logger: logger).start
+  end
+
+  Gitlab::Ci::Parsers.instrument!
+rescue IOError => e
+  Gitlab::ErrorTracking.track_exception(e)
+  Gitlab::Metrics.error_detected!
+end
+
+if Gitlab::Runtime.puma?
+  Gitlab::Cluster::LifecycleEvents.on_before_graceful_shutdown do
+    # We need to ensure that before we re-exec or shutdown server
+    # we do stop the exporter
+    Gitlab::Metrics::Exporter::WebExporter.instance.stop
+  end
+
+  Gitlab::Cluster::LifecycleEvents.on_before_master_restart do
+    # We need to ensure that before we re-exec server
+    # we do stop the exporter
+    #
+    # We do it again, for being extra safe,
+    # but it should not be needed
+    Gitlab::Metrics::Exporter::WebExporter.instance.stop
+  end
+end

doc/architecture/blueprints/composable_codebase_using_rails_engines/index.md

@@ -383,7 +383,7 @@ What was done?
     ```ruby
     # config/engines.rb
     # Load only in case we are running web_server or rails console
-    if Gitlab::Runtime.web_server? || Gitlab::Runtime.console?
+    if Gitlab::Runtime.puma? || Gitlab::Runtime.console?
       require 'web_engine'
     end
     ```

lib/gitlab/gitaly_client.rb

@@ -390,7 +390,7 @@ module Gitlab
     end
 
     def self.long_timeout
-      if Gitlab::Runtime.web_server?
+      if Gitlab::Runtime.puma?
         default_timeout
       else
         6.hours

lib/gitlab/pages/settings.rb

@@ -16,7 +16,7 @@ module Gitlab
       def disk_access_denied?
         return true unless ::Settings.pages.local_store&.enabled
 
-        ::Gitlab::Runtime.web_server? && !::Gitlab::Runtime.test_suite?
+        ::Gitlab::Runtime.puma? && !::Gitlab::Runtime.test_suite?
       end
 
       def report_denied_disk_access

lib/gitlab/runtime.rb

@@ -65,12 +65,15 @@ module Gitlab
         !!defined?(::Rails::Command::RunnerCommand)
       end
 
-      def web_server?
-        puma?
+      # Whether we are executing in an actual application context i.e. Puma or Sidekiq.
+      def application?
+        puma? || sidekiq?
       end
 
+      # Whether we are executing in a multi-threaded environment. For now this is equivalent
+      # to meaning Puma or Sidekiq, but this could change in the future.
       def multi_threaded?
-        puma? || sidekiq?
+        application?
       end
 
       def puma_in_clustered_mode?
@@ -94,7 +97,7 @@ module Gitlab
           threads += Sidekiq.options[:concurrency] + 2
         end
 
-        if web_server?
+        if puma?
           threads += Gitlab::ActionCable::Config.worker_pool_size
         end
 

spec/lib/gitlab/pages/settings_spec.rb

@@ -13,7 +13,7 @@ RSpec.describe Gitlab::Pages::Settings do
     context 'when running under a web server outside of test mode' do
       before do
         allow(::Gitlab::Runtime).to receive(:test_suite?).and_return(false)
-        allow(::Gitlab::Runtime).to receive(:web_server?).and_return(true)
+        allow(::Gitlab::Runtime).to receive(:puma?).and_return(true)
       end
 
       it 'logs a DiskAccessDenied error' do

spec/lib/gitlab/runtime_spec.rb

@@ -80,6 +80,10 @@ RSpec.describe Gitlab::Runtime do
 
     it_behaves_like "valid runtime", :puma, 3 + Gitlab::ActionCable::Config.worker_pool_size
 
+    it 'identifies as an application runtime' do
+      expect(Gitlab::Runtime.application?).to be true
+    end
+
     context "when ActionCable worker pool size is configured" do
       before do
         stub_env('ACTION_CABLE_WORKER_POOL_SIZE', 10)
@@ -113,6 +117,10 @@ RSpec.describe Gitlab::Runtime do
     end
 
     it_behaves_like "valid runtime", :sidekiq, 5
+
+    it 'identifies as an application runtime' do
+      expect(Gitlab::Runtime.application?).to be true
+    end
   end
 
   context "console" do
@@ -121,6 +129,10 @@ RSpec.describe Gitlab::Runtime do
     end
 
     it_behaves_like "valid runtime", :console, 1
+
+    it 'does not identify as an application runtime' do
+      expect(Gitlab::Runtime.application?).to be false
+    end
   end
 
   context "test suite" do
@@ -129,6 +141,10 @@ RSpec.describe Gitlab::Runtime do
     end
 
     it_behaves_like "valid runtime", :test_suite, 1
+
+    it 'does not identify as an application runtime' do
+      expect(Gitlab::Runtime.application?).to be false
+    end
   end
 
   context "geo log cursor" do
@@ -145,5 +161,9 @@ RSpec.describe Gitlab::Runtime do
     end
 
     it_behaves_like "valid runtime", :rails_runner, 1
+
+    it 'does not identify as an application runtime' do
+      expect(Gitlab::Runtime.application?).to be false
+    end
   end
 end