Merge branch 'mc/maintenance/spread-ci-minutes-reset' into 'master'

Spread CI minute reset workers over 3 hours

See merge request gitlab-org/gitlab!46927

ee/app/workers/clear_shared_runners_minutes_worker.rb

@@ -17,9 +17,17 @@ class ClearSharedRunnersMinutesWorker # rubocop:disable Scalability/IdempotentWo
       start_id = Namespace.minimum(:id)
       last_id = Namespace.maximum(:id)
 
-      (start_id..last_id).step(BATCH_SIZE) do |batch_start_id|
+      execution_offset = 3.hours.seconds / ((last_id - start_id) / BATCH_SIZE)
+
+      (start_id..last_id).step(BATCH_SIZE).with_index do |batch_start_id, batch_index|
         batch_end_id = batch_start_id + BATCH_SIZE - 1
-        Ci::BatchResetMinutesWorker.perform_async(batch_start_id, batch_end_id)
+
+        delay = execution_offset * batch_index
+
+        # #perform_in is used instead of #perform_async to spread the load
+        # evenly accross the first three hours of the month to avoid stressing
+        # the database.
+        Ci::BatchResetMinutesWorker.perform_in(delay, batch_start_id, batch_end_id)
       end
     else
       return unless try_obtain_lease

ee/changelogs/unreleased/mc-maintenance-spread-ci-minutes-reset.yml

+---
+title: Spread CI minute reset over 3 hours at start of each month.
+merge_request: 46927
+author:
+type: performance

ee/spec/workers/clear_shared_runners_minutes_worker_spec.rb

@@ -147,11 +147,12 @@ RSpec.describe ClearSharedRunnersMinutesWorker do
       end
 
       it 'runs a worker per batch' do
-        expect(Ci::BatchResetMinutesWorker).to receive(:perform_async).with(2, 4)
-        expect(Ci::BatchResetMinutesWorker).to receive(:perform_async).with(5, 7)
-        expect(Ci::BatchResetMinutesWorker).to receive(:perform_async).with(8, 10)
-        expect(Ci::BatchResetMinutesWorker).to receive(:perform_async).with(11, 13)
-        expect(Ci::BatchResetMinutesWorker).to receive(:perform_async).with(14, 16)
+        # Spread evenly accross 3 hours (10800 seconds)
+        expect(Ci::BatchResetMinutesWorker).to receive(:perform_in).with(0.seconds, 2, 4)
+        expect(Ci::BatchResetMinutesWorker).to receive(:perform_in).with(2700.seconds, 5, 7)
+        expect(Ci::BatchResetMinutesWorker).to receive(:perform_in).with(5400.seconds, 8, 10)
+        expect(Ci::BatchResetMinutesWorker).to receive(:perform_in).with(8100.seconds, 11, 13)
+        expect(Ci::BatchResetMinutesWorker).to receive(:perform_in).with(10800.seconds, 14, 16)
 
         subject
       end