Commit 65be6f1a authored by Kamil Trzciński's avatar Kamil Trzciński

Merge branch 'puma-mem-counter' into 'master'

Add puma killer observer

Closes #63037

See merge request gitlab-org/gitlab-ce!28992
parents d5e8e1ef 497acb16
...@@ -42,7 +42,6 @@ bind 'unix:///home/git/gitlab.socket' ...@@ -42,7 +42,6 @@ bind 'unix:///home/git/gitlab.socket'
workers 2 workers 2
require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events" require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer"
on_restart do on_restart do
# Signal application hooks that we're about to restart # Signal application hooks that we're about to restart
......
...@@ -125,6 +125,7 @@ When Puma is used instead of Unicorn, following metrics are available: ...@@ -125,6 +125,7 @@ When Puma is used instead of Unicorn, following metrics are available:
| puma_max_threads | Gauge | 12.0 | Maximum number of worker threads | | puma_max_threads | Gauge | 12.0 | Maximum number of worker threads |
| puma_idle_threads | Gauge | 12.0 | Number of spawned threads which are not processing a request | | puma_idle_threads | Gauge | 12.0 | Number of spawned threads which are not processing a request |
| rack_state_total | Gauge | 12.0 | Number of requests in a given rack state | | rack_state_total | Gauge | 12.0 | Number of requests in a given rack state |
| puma_killer_terminations_total | Gauge | 12.0 | Number of workers terminated by PumaWorkerKiller |
## Metrics shared directory ## Metrics shared directory
......
...@@ -27,6 +27,9 @@ module Gitlab ...@@ -27,6 +27,9 @@ module Gitlab
# is restarted already, thus periodically restarting workers shouldn't be # is restarted already, thus periodically restarting workers shouldn't be
# needed. # needed.
config.rolling_restart_frequency = false config.rolling_restart_frequency = false
observer = Gitlab::Cluster::PumaWorkerKillerObserver.new
config.pre_term = observer.callback
end end
PumaWorkerKiller.start PumaWorkerKiller.start
......
# frozen_string_literal: true
module Gitlab
module Cluster
class PumaWorkerKillerObserver
def initialize
@counter = Gitlab::Metrics.counter(:puma_killer_terminations_total, 'Number of workers terminated by PumaWorkerKiller')
end
# returns the Proc to be used as the observer callback block
def callback
method(:log_termination)
end
private
def log_termination(worker)
labels = { worker: "worker_#{worker.index}" }
@counter.increment(labels)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Cluster::PumaWorkerKillerObserver do
let(:counter) { Gitlab::Metrics::NullMetric.instance }
before do
allow(Gitlab::Metrics).to receive(:counter)
.with(any_args)
.and_return(counter)
end
describe '#callback' do
subject { described_class.new }
it 'increments timeout counter' do
worker = double(index: 0)
expect(counter)
.to receive(:increment)
.with({ worker: 'worker_0' })
subject.callback.call(worker)
end
end
end
...@@ -20,7 +20,7 @@ describe 'Puma' do ...@@ -20,7 +20,7 @@ describe 'Puma' do
File.write(config_path, config_lines) File.write(config_path, config_lines)
cmd = %W[puma -e test -C #{config_path} #{File.join(__dir__, 'configs/config.ru')}] cmd = %W[puma -e test -C #{config_path} #{File.join(__dir__, 'configs/config.ru')}]
@puma_master_pid = spawn(*cmd) @puma_master_pid = spawn({ 'DISABLE_PUMA_WORKER_KILLER' => '1' }, *cmd)
wait_puma_boot!(@puma_master_pid, File.join(project_root, 'tmp/tests/puma-worker-ready')) wait_puma_boot!(@puma_master_pid, File.join(project_root, 'tmp/tests/puma-worker-ready'))
WebMock.allow_net_connect! WebMock.allow_net_connect!
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment