Add metric for measuring PumaWorkerKiller activity

PumaWorkerKiller is used for periodically checking and killing
workers (the biggest one) if overall memory reaches specified
limit. This metric allows us to watch number of killed workers.

config/puma.example.development.rb

@@ -42,7 +42,6 @@ bind 'unix:///home/git/gitlab.socket'
 workers 2
 
 require_relative "/home/git/gitlab/lib/gitlab/cluster/lifecycle_events"
-require_relative "/home/git/gitlab/lib/gitlab/cluster/puma_worker_killer_initializer"
 
 on_restart do
   # Signal application hooks that we're about to restart

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -125,6 +125,7 @@ When Puma is used instead of Unicorn, following metrics are available:
 | puma_max_threads                             | Gauge   | 12.0  | Maximum number of worker threads |
 | puma_idle_threads                            | Gauge   | 12.0  | Number of spawned threads which are not processing a request |
 | rack_state_total                             | Gauge   | 12.0  | Number of requests in a given rack state |
+| puma_killer_terminations_total               | Gauge   | 12.0  | Number of workers terminated by PumaWorkerKiller |
 
 ## Metrics shared directory
 

lib/gitlab/cluster/puma_worker_killer_initializer.rb

@@ -27,6 +27,9 @@ module Gitlab
           # is restarted already, thus periodically restarting workers shouldn't be
           # needed.
           config.rolling_restart_frequency = false
+
+          observer = Gitlab::Cluster::PumaWorkerKillerObserver.new
+          config.pre_term = observer.callback
         end
 
         PumaWorkerKiller.start

lib/gitlab/cluster/puma_worker_killer_observer.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Cluster
+    class PumaWorkerKillerObserver
+      def initialize
+        @counter = Gitlab::Metrics.counter(:puma_killer_terminations_total, 'Number of workers terminated by PumaWorkerKiller')
+      end
+
+      # returns the Proc to be used as the observer callback block
+      def callback
+        method(:log_termination)
+      end
+
+      private
+
+      def log_termination(worker)
+        labels = { worker: "worker_#{worker.index}" }
+
+        @counter.increment(labels)
+      end
+    end
+  end
+end

spec/lib/gitlab/cluster/puma_worker_killer_observer_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::Cluster::PumaWorkerKillerObserver do
+  let(:counter) { Gitlab::Metrics::NullMetric.instance }
+
+  before do
+    allow(Gitlab::Metrics).to receive(:counter)
+      .with(any_args)
+      .and_return(counter)
+  end
+
+  describe '#callback' do
+    subject { described_class.new }
+
+    it 'increments timeout counter' do
+      worker = double(index: 0)
+
+      expect(counter)
+        .to receive(:increment)
+        .with({ worker: 'worker_0' })
+
+      subject.callback.call(worker)
+    end
+  end
+end

spec/rack_servers/puma_spec.rb

@@ -20,7 +20,7 @@ describe 'Puma' do
     File.write(config_path, config_lines)
 
     cmd = %W[puma -e test -C #{config_path} #{File.join(__dir__, 'configs/config.ru')}]
-    @puma_master_pid = spawn(*cmd)
+    @puma_master_pid = spawn({ 'DISABLE_PUMA_WORKER_KILLER' => '1' }, *cmd)
     wait_puma_boot!(@puma_master_pid, File.join(project_root, 'tmp/tests/puma-worker-ready'))
     WebMock.allow_net_connect!
   end