Commit 581b3020 authored by Matthias Kaeppler's avatar Matthias Kaeppler

Revert revert of 35170-consolidate-process-config

This reverts commit b0944b95.
parent 7a8057fd
......@@ -163,7 +163,7 @@ module Git
end
def logger
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
Sidekiq.logger
else
# This service runs in Sidekiq, so this shouldn't ever be
......
......@@ -22,6 +22,7 @@ module Gitlab
require_dependency Rails.root.join('lib/gitlab/current_settings')
require_dependency Rails.root.join('lib/gitlab/middleware/read_only')
require_dependency Rails.root.join('lib/gitlab/middleware/basic_health_check')
require_dependency Rails.root.join('lib/gitlab/runtime')
# Settings in config/environments/* take precedence over those specified here.
# Application configuration should go into files in config/initializers
......@@ -255,7 +256,7 @@ module Gitlab
caching_config_hash[:compress] = false
caching_config_hash[:namespace] = Gitlab::Redis::Cache::CACHE_NAMESPACE
caching_config_hash[:expires_in] = 2.weeks # Cache should not grow forever
if Sidekiq.server? || defined?(::Puma) # threaded context
if Gitlab::Runtime.multi_threaded?
caching_config_hash[:pool_size] = Gitlab::Redis::Cache.pool_size
caching_config_hash[:pool_timeout] = 1
end
......
......@@ -46,7 +46,7 @@ Rails.application.configure do
# Do not log asset requests
config.assets.quiet = true
config.allow_concurrency = defined?(::Puma)
config.allow_concurrency = Gitlab::Runtime.multi_threaded?
# BetterErrors live shell (REPL) on every stack frame
BetterErrors::Middleware.allow_ip!("127.0.0.1/0")
......
......@@ -75,5 +75,5 @@ Rails.application.configure do
config.eager_load = true
config.allow_concurrency = defined?(::Puma)
config.allow_concurrency = Gitlab::Runtime.multi_threaded?
end
# frozen_string_literal: true
begin
Gitlab::AppLogger.info("Runtime: #{Gitlab::Runtime.name}")
rescue => e
message = <<-NOTICE
\n!! RUNTIME IDENTIFICATION FAILED: #{e}
Runtime based configuration settings may not work properly.
If you continue to see this error, please file an issue via
https://gitlab.com/gitlab-org/gitlab/issues/new
NOTICE
Gitlab::AppLogger.error(message)
end
......@@ -364,7 +364,7 @@ Gitlab.ee do
# To ensure acceptable performance we only allow feature to be used with
# multithreaded web-server Puma. This will be removed once download logic is moved
# to GitLab workhorse
Settings.dependency_proxy['enabled'] = false unless defined?(::Puma)
Settings.dependency_proxy['enabled'] = false unless Gitlab::Runtime.puma?
end
#
......
......@@ -4,11 +4,11 @@ require 'prometheus/client'
def prometheus_default_multiproc_dir
return unless Rails.env.development? || Rails.env.test?
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
Rails.root.join('tmp/prometheus_multiproc_dir/sidekiq')
elsif defined?(Unicorn::Worker)
elsif Gitlab::Runtime.unicorn?
Rails.root.join('tmp/prometheus_multiproc_dir/unicorn')
elsif defined?(::Puma)
elsif Gitlab::Runtime.puma?
Rails.root.join('tmp/prometheus_multiproc_dir/puma')
else
Rails.root.join('tmp/prometheus_multiproc_dir')
......@@ -48,9 +48,9 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
Gitlab::Cluster::LifecycleEvents.on_master_start do
::Prometheus::Client.reinitialize_on_pid_change(force: true)
if defined?(::Unicorn)
if Gitlab::Runtime.unicorn?
Gitlab::Metrics::Samplers::UnicornSampler.instance(Settings.monitoring.unicorn_sampler_interval).start
elsif defined?(::Puma)
elsif Gitlab::Runtime.puma?
Gitlab::Metrics::Samplers::PumaSampler.instance(Settings.monitoring.puma_sampler_interval).start
end
......@@ -58,7 +58,7 @@ if !Rails.env.test? && Gitlab::Metrics.prometheus_metrics_enabled?
end
end
if defined?(::Unicorn) || defined?(::Puma)
if Gitlab::Runtime.app_server?
Gitlab::Cluster::LifecycleEvents.on_master_start do
Gitlab::Metrics::Exporter::WebExporter.instance.start
end
......
......@@ -2,7 +2,7 @@
# Don't handle sidekiq configuration as it
# has its own special active record configuration here
if defined?(ActiveRecord::Base) && !Sidekiq.server?
if defined?(ActiveRecord::Base) && !Gitlab::Runtime.sidekiq?
Gitlab::Cluster::LifecycleEvents.on_worker_start do
ActiveSupport.on_load(:active_record) do
ActiveRecord::Base.establish_connection
......
......@@ -5,10 +5,8 @@
#
# Follow-up the issue: https://gitlab.com/gitlab-org/gitlab/issues/34107
if defined?(::Puma)
if Gitlab::Runtime.puma?
Puma::Cluster.prepend(::Gitlab::Cluster::Mixins::PumaCluster)
end
if defined?(::Unicorn::HttpServer)
elsif Gitlab::Runtime.unicorn?
Unicorn::HttpServer.prepend(::Gitlab::Cluster::Mixins::UnicornHttpServer)
end
......@@ -2,7 +2,7 @@
# when running on puma, scale connection pool size with the number
# of threads per worker process
if defined?(::Puma)
if Gitlab::Runtime.puma?
db_config = Gitlab::Database.config ||
Rails.application.config.database_configuration[Rails.env]
puma_options = Puma.cli_config.options
......
# Only use Lograge for Rails
unless Sidekiq.server?
unless Gitlab::Runtime.sidekiq?
filename = File.join(Rails.root, 'log', "#{Rails.env}_json.log")
Rails.application.configure do
......
......@@ -9,7 +9,7 @@
# and it's used only as the last resort. In such case this termination is
# logged and we should fix the potential timeout issue in the code itself.
if defined?(::Puma) && !Rails.env.test?
if Gitlab::Runtime.puma? && !Rails.env.test?
require 'rack/timeout/base'
Gitlab::Application.configure do |config|
......
......@@ -13,7 +13,7 @@ if Labkit::Tracing.enabled?
end
# Instrument Sidekiq server calls when running Sidekiq server
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
Sidekiq.configure_server do |config|
config.server_middleware do |chain|
chain.add Labkit::Tracing::Sidekiq::ServerMiddleware
......
# frozen_string_literal: true
if defined?(::Puma) && ::Puma.cli_config.options[:workers].to_i.zero?
if Gitlab::Runtime.puma? && ::Puma.cli_config.options[:workers].to_i.zero?
raise 'Puma is only supported in Cluster-mode: workers > 0'
end
......@@ -85,7 +85,7 @@ module Gitlab
# Returns true if load balancing is to be enabled.
def self.enable?
return false unless ::License.feature_available?(:db_load_balancing)
return false if program_name == 'rake' || Sidekiq.server?
return false if program_name == 'rake' || Gitlab::Runtime.sidekiq?
hosts.any? || service_discovery_enabled?
end
......
......@@ -140,7 +140,7 @@ describe Gitlab::Database::LoadBalancing do
it 'returns false when Sidekiq is being used' do
allow(described_class).to receive(:hosts).and_return(%w(foo))
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
expect(described_class.enable?).to eq(false)
end
......@@ -153,14 +153,14 @@ describe Gitlab::Database::LoadBalancing do
it 'returns true when load balancing should be enabled' do
allow(described_class).to receive(:hosts).and_return(%w(foo))
allow(Sidekiq).to receive(:server?).and_return(false)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
expect(described_class.enable?).to eq(true)
end
it 'returns true when service discovery is enabled' do
allow(described_class).to receive(:hosts).and_return([])
allow(Sidekiq).to receive(:server?).and_return(false)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
allow(described_class)
.to receive(:service_discovery_enabled?)
......@@ -192,7 +192,7 @@ describe Gitlab::Database::LoadBalancing do
it 'is enabled' do
allow(described_class).to receive(:hosts).and_return(%w(foo))
allow(Sidekiq).to receive(:server?).and_return(false)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
expect(described_class.enable?).to eq(true)
end
......
......@@ -100,8 +100,8 @@ module Gitlab
end
def self.process_name
return 'sidekiq' if Sidekiq.server?
return 'console' if defined?(Rails::Console)
return 'sidekiq' if Gitlab::Runtime.sidekiq?
return 'console' if Gitlab::Runtime.console?
return 'test' if Rails.env.test?
'web'
......
......@@ -149,10 +149,10 @@ module Gitlab
def in_clustered_environment?
# Sidekiq doesn't fork
return false if Sidekiq.server?
return false if Gitlab::Runtime.sidekiq?
# Unicorn always forks
return true if defined?(::Unicorn)
return true if Gitlab::Runtime.unicorn?
# Puma sometimes forks
return true if in_clustered_puma?
......@@ -162,7 +162,7 @@ module Gitlab
end
def in_clustered_puma?
return false unless defined?(::Puma)
return false unless Gitlab::Runtime.puma?
@puma_options && @puma_options[:workers] && @puma_options[:workers] > 0
end
......
......@@ -29,7 +29,7 @@ module Gitlab
PEM_REGEX = /\-+BEGIN CERTIFICATE\-+.+?\-+END CERTIFICATE\-+/m.freeze
SERVER_VERSION_FILE = 'GITALY_SERVER_VERSION'
MAXIMUM_GITALY_CALLS = 30
CLIENT_NAME = (Sidekiq.server? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
CLIENT_NAME = (Gitlab::Runtime.sidekiq? ? 'gitlab-sidekiq' : 'gitlab-web').freeze
GITALY_METADATA_FILENAME = '.gitaly-metadata'
MUTEX = Mutex.new
......@@ -382,17 +382,13 @@ module Gitlab
end
def self.long_timeout
if web_app_server?
if Gitlab::Runtime.app_server?
default_timeout
else
6.hours
end
end
def self.web_app_server?
defined?(::Unicorn) || defined?(::Puma)
end
def self.storage_metadata_file_path(storage)
Gitlab::GitalyClient::StorageSettings.allow_disk_access do
File.join(
......
......@@ -135,7 +135,7 @@ module Gitlab
end
def cleanup_time
Sidekiq.server? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S
Gitlab::Runtime.sidekiq? ? BG_CLEANUP_RUNTIME_S : FG_CLEANUP_RUNTIME_S
end
def tmp_keychains_created
......
......@@ -18,7 +18,7 @@ module Gitlab
end
def check
return unless defined?(::Puma)
return unless Gitlab::Runtime.puma?
stats = Puma.stats
stats = JSON.parse(stats)
......
......@@ -30,7 +30,7 @@ module Gitlab
# to change so we can cache the list of servers.
def http_servers
strong_memoize(:http_servers) do
next unless defined?(::Unicorn::HttpServer)
next unless Gitlab::Runtime.unicorn?
ObjectSpace.each_object(::Unicorn::HttpServer).to_a
end
......
......@@ -68,7 +68,7 @@ module Gitlab
end
def timeout_time
Sidekiq.server? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND
Gitlab::Runtime.sidekiq? ? TIMEOUT_BACKGROUND : TIMEOUT_FOREGROUND
end
def link_dependencies(text, highlighted_text)
......
......@@ -150,7 +150,7 @@ module Gitlab
# Returns the prefix to use for the name of a series.
def series_prefix
@series_prefix ||= Sidekiq.server? ? 'sidekiq_' : 'rails_'
@series_prefix ||= Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_'
end
# Allow access from other metrics related middlewares
......
......@@ -39,14 +39,10 @@ module Gitlab
end
def add_metric(series, values, tags = {})
prefix = sidekiq? ? 'sidekiq_' : 'rails_'
prefix = Gitlab::Runtime.sidekiq? ? 'sidekiq_' : 'rails_'
@metrics << Metric.new("#{prefix}#{series}", values, tags)
end
def sidekiq?
Sidekiq.server?
end
end
end
end
......
......@@ -61,7 +61,7 @@ module Gitlab
# it takes around 80ms. The instances of HttpServers are not a subject
# to change so we can cache the list of servers.
def http_servers
return [] unless defined?(::Unicorn::HttpServer)
return [] unless Gitlab::Runtime.unicorn?
@http_servers ||= ObjectSpace.each_object(::Unicorn::HttpServer).to_a
end
......
......@@ -22,10 +22,10 @@ module Gitlab
def pool_size
# heuristic constant 5 should be a config setting somewhere -- related to CPU count?
size = 5
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
# the pool will be used in a multi-threaded context
size += Sidekiq.options[:concurrency]
elsif defined?(::Puma)
elsif Gitlab::Runtime.puma?
size += Puma.cli_config.options[:max_threads]
end
......
# frozen_string_literal: true
module Gitlab
# Provides routines to identify the current runtime as which the application
# executes, such as whether it is an application server and which one.
module Runtime
class << self
def name
matches = []
matches << :puma if puma?
matches << :unicorn if unicorn?
matches << :console if console?
matches << :sidekiq if sidekiq?
raise "Ambiguous process match: #{matches}" if matches.size > 1
matches.first || :unknown
end
def puma?
!!(defined?(::Puma) && bin == 'puma')
end
# For unicorn, we need to check for actual server instances to avoid false positives.
def unicorn?
!!(defined?(::Unicorn) && defined?(::Unicorn::HttpServer))
end
def sidekiq?
!!(defined?(::Sidekiq) && Sidekiq.server? && bin == 'sidekiq')
end
def console?
!!defined?(::Rails::Console)
end
def app_server?
puma? || unicorn?
end
def multi_threaded?
puma? || sidekiq?
end
private
# Some example values from my system:
# puma: /data/cache/bundle-2.5/bin/puma
# unicorn: unicorn_rails master -E development -c /tmp/unicorn.rb -l 0.0.0.0:8080
# sidekiq: /data/cache/bundle-2.5/bin/sidekiq
# thin: bin/rails
# console: bin/rails
def script_name
$0
end
def bin
File.basename(script_name)
end
end
end
end
......@@ -5,11 +5,11 @@ module Prometheus
extend self
def worker_id
if Sidekiq.server?
if Gitlab::Runtime.sidekiq?
sidekiq_worker_id
elsif defined?(Unicorn::Worker)
elsif Gitlab::Runtime.unicorn?
unicorn_worker_id
elsif defined?(::Puma)
elsif Gitlab::Runtime.puma?
puma_worker_id
else
unknown_process_id
......
......@@ -16,6 +16,7 @@ describe 'Database config initializer' do
let(:puma_options) { { max_threads: 8 } }
before do
allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
stub_const("Puma", puma)
allow(puma).to receive_message_chain(:cli_config, :options).and_return(puma_options)
end
......
......@@ -26,7 +26,7 @@ describe Gitlab::GitalyClient do
context 'running in Unicorn' do
before do
stub_const('Unicorn', 1)
allow(Gitlab::Runtime).to receive(:unicorn?).and_return(true)
end
it { expect(subject.long_timeout).to eq(55) }
......@@ -34,7 +34,7 @@ describe Gitlab::GitalyClient do
context 'running in Puma' do
before do
stub_const('Puma', 1)
allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
end
it { expect(subject.long_timeout).to eq(55) }
......
......@@ -236,7 +236,7 @@ describe Gitlab::Gpg do
context 'when running in Sidekiq' do
before do
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
end
it_behaves_like 'multiple deletion attempts of the tmp-dir', described_class::BG_CLEANUP_RUNTIME_S
......
......@@ -22,6 +22,7 @@ describe Gitlab::HealthChecks::PumaCheck do
context 'when Puma is not loaded' do
before do
allow(Gitlab::Runtime).to receive(:puma?).and_return(false)
hide_const('Puma')
end
......@@ -33,6 +34,7 @@ describe Gitlab::HealthChecks::PumaCheck do
context 'when Puma is loaded' do
before do
allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
stub_const('Puma', Module.new)
end
......
......@@ -26,6 +26,7 @@ describe Gitlab::HealthChecks::UnicornCheck do
context 'when Unicorn is not loaded' do
before do
allow(Gitlab::Runtime).to receive(:unicorn?).and_return(false)
hide_const('Unicorn')
end
......@@ -39,6 +40,7 @@ describe Gitlab::HealthChecks::UnicornCheck do
let(:http_server_class) { Struct.new(:worker_processes) }
before do
allow(Gitlab::Runtime).to receive(:unicorn?).and_return(true)
stub_const('Unicorn::HttpServer', http_server_class)
end
......
......@@ -111,7 +111,7 @@ describe Gitlab::Highlight do
end
it 'utilizes longer timeout for sidekiq' do
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
expect(Timeout).to receive(:timeout).with(described_class::TIMEOUT_BACKGROUND).and_call_original
subject.highlight("Content")
......
......@@ -63,7 +63,7 @@ describe Gitlab::Metrics::Samplers::InfluxSampler do
describe '#add_metric' do
it 'prefixes the series name for a Rails process' do
expect(sampler).to receive(:sidekiq?).and_return(false)
expect(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
expect(Gitlab::Metrics::Metric).to receive(:new)
.with('rails_cats', { value: 10 }, {})
......@@ -73,7 +73,7 @@ describe Gitlab::Metrics::Samplers::InfluxSampler do
end
it 'prefixes the series name for a Sidekiq process' do
expect(sampler).to receive(:sidekiq?).and_return(true)
expect(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
expect(Gitlab::Metrics::Metric).to receive(:new)
.with('sidekiq_cats', { value: 10 }, {})
......
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Runtime do
REAL_PATH = $0
after(:all) do
$0 = REAL_PATH
end
context "when unknown" do
it "identifies as :unknown" do
expect(subject.name).to eq(:unknown)
end
end
context "on multiple matches" do
before do
$0 = '/data/cache/bundle-2.5/bin/puma'
stub_const('::Puma', double)
stub_const('::Rails::Console', double)
end
it "raises an exception when trying to identify" do
expect { subject.name }.to raise_error(RuntimeError, "Ambiguous process match: [:puma, :console]")
end
end
context "puma" do
let(:puma_type) { double('::Puma') }
before do
$0 = '/data/cache/bundle-2.5/bin/puma'
stub_const('::Puma', puma_type)
end
it "identifies itself" do
expect(subject.name).to eq(:puma)
expect(subject.puma?).to be(true)
end
it "does not identify as others" do
expect(subject.unicorn?).to be(false)
expect(subject.sidekiq?).to be(false)
expect(subject.console?).to be(false)
end
end
context "unicorn" do
let(:unicorn_type) { Module.new }
let(:unicorn_server_type) { Class.new }
before do
$0 = 'unicorn_rails master -E development -c /tmp/unicorn.rb -l 0.0.0.0:8080'
stub_const('::Unicorn', unicorn_type)
stub_const('::Unicorn::HttpServer', unicorn_server_type)
end
it "identifies itself" do
expect(subject.name).to eq(:unicorn)
expect(subject.unicorn?).to be(true)
end
it "does not identify as others" do
expect(subject.puma?).to be(false)
expect(subject.sidekiq?).to be(false)
expect(subject.console?).to be(false)
end
end
context "sidekiq" do
let(:sidekiq_type) { double('::Sidekiq') }
before do
$0 = '/data/cache/bundle-2.5/bin/sidekiq'
stub_const('::Sidekiq', sidekiq_type)
allow(sidekiq_type).to receive(:server?).and_return(true)
end
it "identifies itself" do
expect(subject.name).to eq(:sidekiq)
expect(subject.sidekiq?).to be(true)
end
it "does not identify as others" do
expect(subject.unicorn?).to be(false)
expect(subject.puma?).to be(false)
expect(subject.console?).to be(false)
end
end
context "console" do
let(:console_type) { double('::Rails::Console') }
before do
$0 = 'bin/rails'
stub_const('::Rails::Console', console_type)
end
it "identifies itself" do
expect(subject.name).to eq(:console)
expect(subject.console?).to be(true)
end
it "does not identify as others" do
expect(subject.unicorn?).to be(false)
expect(subject.sidekiq?).to be(false)
expect(subject.puma?).to be(false)
end
end
end
......@@ -6,16 +6,13 @@ describe Prometheus::PidProvider do
describe '.worker_id' do
subject { described_class.worker_id }
let(:sidekiq_module) { Module.new }
before do
allow(sidekiq_module).to receive(:server?).and_return(false)
stub_const('Sidekiq', sidekiq_module)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
end
context 'when running in Sidekiq server mode' do
before do
expect(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
end
context 'in a clustered setup' do
......@@ -33,8 +30,7 @@ describe Prometheus::PidProvider do
context 'when running in Unicorn mode' do
before do
stub_const('Unicorn::Worker', Class.new)
hide_const('Puma')
allow(Gitlab::Runtime).to receive(:unicorn?).and_return(true)
expect(described_class).to receive(:process_name)
.at_least(:once)
......@@ -94,8 +90,7 @@ describe Prometheus::PidProvider do
context 'when running in Puma mode' do
before do
stub_const('Puma', Module.new)
hide_const('Unicorn::Worker')
allow(Gitlab::Runtime).to receive(:puma?).and_return(true)
expect(described_class).to receive(:process_name)
.at_least(:once)
......@@ -116,11 +111,6 @@ describe Prometheus::PidProvider do
end
context 'when running in unknown mode' do
before do
hide_const('Puma')
hide_const('Unicorn::Worker')
end
it { is_expected.to eq "process_#{Process.pid}" }
end
end
......
......@@ -108,7 +108,7 @@ describe Git::BranchPushService, services: true do
end
it 'reports an error' do
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
expect(Sidekiq.logger).to receive(:warn)
expect { subject }.not_to change { Ci::Pipeline.count }
......
......@@ -118,7 +118,7 @@ RSpec.shared_examples "redis_shared_examples" do
context 'when running not on sidekiq workers' do
before do
allow(Sidekiq).to receive(:server?).and_return(false)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(false)
end
it 'instantiates a connection pool with size 5' do
......@@ -130,7 +130,7 @@ RSpec.shared_examples "redis_shared_examples" do
context 'when running on sidekiq workers' do
before do
allow(Sidekiq).to receive(:server?).and_return(true)
allow(Gitlab::Runtime).to receive(:sidekiq?).and_return(true)
allow(Sidekiq).to receive(:options).and_return({ concurrency: 18 })
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment