From 4f9a14061b707f39d8a98dae328089bbfbc09e70 Mon Sep 17 00:00:00 2001 From: Jacob Vosmaer <contact@jacobvosmaer.nl> Date: Fri, 5 Dec 2014 17:29:34 +0100 Subject: [PATCH] Wait 15 minutes before Sidekiq MemoryKiller action --- .../sidekiq_middleware/memory_killer.rb | 48 +++++++++++++------ 1 file changed, 33 insertions(+), 15 deletions(-) diff --git a/lib/gitlab/sidekiq_middleware/memory_killer.rb b/lib/gitlab/sidekiq_middleware/memory_killer.rb index 0fb09d3f228..df8968cf677 100644 --- a/lib/gitlab/sidekiq_middleware/memory_killer.rb +++ b/lib/gitlab/sidekiq_middleware/memory_killer.rb @@ -1,26 +1,38 @@ module Gitlab module SidekiqMiddleware class MemoryKiller + # Give Sidekiq 15 minutes of grace time after exceeding the RSS limit + GRACE_TIME = 15 * 60 # Wait 30 seconds for running jobs to finish during graceful shutdown - GRACEFUL_SHUTDOWN_WAIT = 30 + SHUTDOWN_WAIT = 30 + # Create a mutex so that there will be only one thread waiting to shut + # Sidekiq down + MUTEX = Mutex.new def call(worker, job, queue) yield current_rss = get_rss + return unless max_rss > 0 && current_rss > max_rss - Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ - "#{max_rss}" - Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" - # SIGUSR1 tells Sidekiq to stop accepting new jobs - Process.kill('SIGUSR1', Process.pid) - - Sidekiq.logger.warn "spawning thread that will send SIGTERM to PID "\ - "#{Process.pid} in #{graceful_shutdown_wait} seconds" - # Send the final shutdown signal to Sidekiq from a separate thread so - # that the current job can finish - Thread.new do - sleep(graceful_shutdown_wait) + Tread.new do + # Return if another thread is already waiting to shut Sidekiq down + return unless MUTEX.try_lock + + Sidekiq.logger.warn "current RSS #{current_rss} exceeds maximum RSS "\ + "#{max_rss}" + Sidekiq.logger.warn "spawned thread that will shut down PID "\ + "#{Process.pid} in #{grace_time} seconds" + sleep(grace_time) + + Sidekiq.logger.warn "sending SIGUSR1 to PID #{Process.pid}" + Process.kill('SIGUSR1', Process.pid) + + Sidekiq.logger.warn "waiting #{shutdown_wait} seconds before sending "\ + "SIGTERM to PID #{Process.pid}" + sleep(shutdown_wait) + + Sidekiq.logger.warn "sending SIGTERM to PID #{Process.pid}" Process.kill('SIGTERM', Process.pid) end end @@ -38,9 +50,15 @@ module Gitlab @max_rss ||= ENV['SIDEKIQ_MAX_RSS'].to_s.to_i end - def graceful_shutdown_wait + def shutdown_wait @graceful_shutdown_wait ||= ( - ENV['SIDEKIQ_GRACEFUL_SHUTDOWN_WAIT'] || GRACEFUL_SHUTDOWN_WAIT + ENV['SIDEKIQ_MEMORY_KILLER_SHUTDOWN_WAIT'] || SHUTDOWN_WAIT + ).to_i + end + + def grace_time + @grace_time ||= ( + ENV['SIDEKIQ_MEMORY_KILLER_GRACE_TIME'] || GRACE_TIME ).to_i end end -- 2.30.9