Commit 9254b5d4 authored by Stan Hu's avatar Stan Hu

Merge branch 'osw-fix-post-dep-migration-with-timeout' into 'master'

Avoid DB timeouts when scheduling migrations

Closes #63036

See merge request gitlab-org/gitlab-ce!29437
parents 65be6f1a 304b43a6
---
title: Avoid DB timeouts when scheduling migrations
merge_request: 29437
author:
type: fixed
...@@ -16,15 +16,10 @@ class EnqueueResetMergeStatus < ActiveRecord::Migration[5.1] ...@@ -16,15 +16,10 @@ class EnqueueResetMergeStatus < ActiveRecord::Migration[5.1]
def up def up
say 'Scheduling `ResetMergeStatus` jobs' say 'Scheduling `ResetMergeStatus` jobs'
# We currently have around 135_000 opened, mergeable MRs in GitLab.com. This iteration # We currently have more than ~5_000_000 merge request records on GitLab.com.
# will schedule around 13 batches of 10_000 MRs, which should take around 1 hour to # This means it'll schedule ~500 jobs (10k MRs each) with a 5 minutes gap,
# complete. # so this should take ~41 hours for all background migrations to complete.
relation = MergeRequest.where(state: 'opened', merge_status: 'can_be_merged') # ((5_000_000 / 10_000) * 5) / 60 => 41.6666..
queue_background_migration_jobs_by_range_at_intervals(MergeRequest, MIGRATION, DELAY_INTERVAL, batch_size: BATCH_SIZE)
relation.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck('MIN(id)', 'MAX(id)').first
BackgroundMigrationWorker.perform_in(index * DELAY_INTERVAL, MIGRATION, range)
end
end end
end end
...@@ -40,9 +40,12 @@ describe EnqueueResetMergeStatus, :migration, :sidekiq do ...@@ -40,9 +40,12 @@ describe EnqueueResetMergeStatus, :migration, :sidekiq do
.to be_scheduled_delayed_migration(5.minutes, 1, 2) .to be_scheduled_delayed_migration(5.minutes, 1, 2)
expect(described_class::MIGRATION) expect(described_class::MIGRATION)
.to be_scheduled_delayed_migration(10.minutes, 3, 3) .to be_scheduled_delayed_migration(10.minutes, 3, 4)
expect(BackgroundMigrationWorker.jobs.size).to eq(2) expect(described_class::MIGRATION)
.to be_scheduled_delayed_migration(15.minutes, 5, 5)
expect(BackgroundMigrationWorker.jobs.size).to eq(3)
end end
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment