Merge branch '249519-fix-daemon-jobs-hash-thread-safe-issue' into 'master'

Fix daemon memory killer jobs hash thread safety issue

Closes #249519

See merge request gitlab-org/gitlab!42468

changelogs/unreleased/249519-fix-daemon-jobs-hash-thread-safe-issue.yml

+---
+title: Fix daemon memory killer jobs hash thread safety issue
+merge_request: 42468
+author:
+type: fixed

lib/gitlab/sidekiq_daemon/memory_killer.rb

@@ -230,10 +230,12 @@ module Gitlab
       end
 
       def rss_increase_by_jobs
+        Gitlab::SidekiqDaemon::Monitor.instance.jobs_mutex.synchronize do
           Gitlab::SidekiqDaemon::Monitor.instance.jobs.sum do |job| # rubocop:disable CodeReuse/ActiveRecord
             rss_increase_by_job(job)
           end
         end
+      end
 
       def rss_increase_by_job(job)
         memory_growth_kb = get_job_options(job, 'memory_killer_memory_growth_kb', 0).to_i

spec/lib/gitlab/sidekiq_daemon/memory_killer_spec.rb

@@ -49,7 +49,7 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do
         expect { subject }.not_to raise_exception
       end
 
-      it 'logs exception message once and raise execption and log stop message' do
+      it 'logs exception message once and raise exception and log stop message' do
         expect(Sidekiq.logger).to receive(:warn).once
           .with(
             class: described_class.to_s,
@@ -68,7 +68,7 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do
             pid: pid,
             message: 'Stopping Gitlab::SidekiqDaemon::MemoryKiller Daemon')
 
-        expect { subject }.to raise_exception
+        expect { subject }.to raise_exception(Exception, 'My Exception')
       end
 
       it 'logs stop message once' do
@@ -402,12 +402,14 @@ RSpec.describe Gitlab::SidekiqDaemon::MemoryKiller do
     subject { memory_killer.send(:rss_increase_by_jobs) }
 
     it 'adds up individual rss_increase_by_job' do
+      allow(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs_mutex, :synchronize).and_yield
       expect(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs).and_return(running_jobs)
       expect(memory_killer).to receive(:rss_increase_by_job).and_return(11, 22)
       expect(subject).to eq(33)
     end
 
     it 'return 0 if no job' do
+      allow(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs_mutex, :synchronize).and_yield
       expect(Gitlab::SidekiqDaemon::Monitor).to receive_message_chain(:instance, :jobs).and_return({})
       expect(subject).to eq(0)
     end