Convert migration to populate latest merge request ID into a background migration

This is to smear updates over a few hours to avoid causing excessive
replication lag as seen in https://gitlab.com/gitlab-com/infrastructure/issues/3235.

db/post_migrate/20171026082505_populate_merge_requests_latest_merge_request_diff_id.rb

-class PopulateMergeRequestsLatestMergeRequestDiffId < ActiveRecord::Migration
-  include Gitlab::Database::MigrationHelpers
-
-  DOWNTIME = false
-  BATCH_SIZE = 1_000
-
-  class MergeRequest < ActiveRecord::Base
-    self.table_name = 'merge_requests'
-
-    include ::EachBatch
-  end
-
-  disable_ddl_transaction!
-
-  def up
-    update = '
-      latest_merge_request_diff_id = (
-        SELECT MAX(id)
-        FROM merge_request_diffs
-        WHERE merge_requests.id = merge_request_diffs.merge_request_id
-      )'.squish
-
-    MergeRequest.where(latest_merge_request_diff_id: nil).each_batch(of: BATCH_SIZE) do |relation|
-      relation.update_all(update)
-    end
-  end
-end

db/post_migrate/20171026082505_schedule_merge_request_latest_merge_request_diff_id_migrations.rb

+class ScheduleMergeRequestLatestMergeRequestDiffIdMigrations < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 50_000
+  MIGRATION = 'PopulateMergeRequestsLatestMergeRequestDiffId'
+
+  disable_ddl_transaction!
+
+  class MergeRequest < ActiveRecord::Base
+    self.table_name = 'merge_requests'
+
+    include ::EachBatch
+  end
+
+  # On GitLab.com, we saw that we generated about 500,000 dead tuples over 5 minutes.
+  # To keep replication lag from ballooning, we'll aim for 50,000 updates over 5 minutes.
+  #
+  # Assuming that there are 5 million rows affected (which is more than on
+  # GitLab.com), and that each batch of 50,000 rows takes up to 5 minutes, then
+  # we can migrate all the rows in 8.5 hours.
+  def up
+    MergeRequest.where(latest_merge_request_diff_id: nil).each_batch(of: BATCH_SIZE) do |relation, index|
+      range = relation.pluck('MIN(id)', 'MAX(id)').first
+
+      BackgroundMigrationWorker.perform_in(index * 5.minutes, MIGRATION, range)
+    end
+  end
+end

lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id.rb

+module Gitlab
+  module BackgroundMigration
+    class PopulateMergeRequestsLatestMergeRequestDiffId
+      BATCH_SIZE = 1_000
+
+      class MergeRequest < ActiveRecord::Base
+        self.table_name = 'merge_requests'
+
+        include ::EachBatch
+      end
+
+      def perform(start_id, stop_id)
+        update = '
+          latest_merge_request_diff_id = (
+            SELECT MAX(id)
+            FROM merge_request_diffs
+            WHERE merge_requests.id = merge_request_diffs.merge_request_id
+          )'.squish
+
+        MergeRequest
+          .where(id: start_id..stop_id)
+          .where(latest_merge_request_diff_id: nil)
+          .each_batch(of: BATCH_SIZE) do |relation|
+
+          relation.update_all(update)
+        end
+      end
+    end
+  end
+end

spec/migrations/populate_merge_requests_latest_merge_request_diff_id_spec.rb → spec/lib/gitlab/background_migration/populate_merge_requests_latest_merge_request_diff_id_spec.rb

 require 'spec_helper'
-require Rails.root.join('db', 'post_migrate', '20171026082505_populate_merge_requests_latest_merge_request_diff_id')
 
-describe PopulateMergeRequestsLatestMergeRequestDiffId, :migration do
+describe Gitlab::BackgroundMigration::PopulateMergeRequestsLatestMergeRequestDiffId, :migration, schema: 20171026082505 do
   let(:projects_table) { table(:projects) }
   let(:merge_requests_table) { table(:merge_requests) }
   let(:merge_request_diffs_table) { table(:merge_request_diffs) }
@@ -27,30 +26,32 @@ describe PopulateMergeRequestsLatestMergeRequestDiffId, :migration do
     merge_request_diffs_table.where(merge_request_id: merge_request.id)
   end
 
-  describe '#up' do
+  describe '#perform' do
     it 'ignores MRs without diffs' do
       merge_request_without_diff = create_mr!('without_diff')
+      mr_id = merge_request_without_diff.id
 
       expect(merge_request_without_diff.latest_merge_request_diff_id).to be_nil
 
-      expect { migrate! }
+      expect { subject.perform(mr_id, mr_id) }
         .not_to change { merge_request_without_diff.reload.latest_merge_request_diff_id }
     end
 
     it 'ignores MRs that have a diff ID already set' do
       merge_request_with_multiple_diffs = create_mr!('with_multiple_diffs', diffs: 3)
       diff_id = diffs_for(merge_request_with_multiple_diffs).minimum(:id)
+      mr_id = merge_request_with_multiple_diffs.id
 
       merge_request_with_multiple_diffs.update!(latest_merge_request_diff_id: diff_id)
 
-      expect { migrate! }
+      expect { subject.perform(mr_id, mr_id) }
         .not_to change { merge_request_with_multiple_diffs.reload.latest_merge_request_diff_id }
     end
 
     it 'migrates multiple MR diffs to the correct values' do
       merge_requests = Array.new(3).map.with_index { |_, i| create_mr!(i, diffs: 3) }
 
-      migrate!
+      subject.perform(merge_requests.first.id, merge_requests.last.id)
 
       merge_requests.each do |merge_request|
         expect(merge_request.reload.latest_merge_request_diff_id)

spec/migrations/schedule_merge_request_latest_merge_request_diff_id_migrations_spec.rb

+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20171026082505_schedule_merge_request_latest_merge_request_diff_id_migrations')
+
+describe ScheduleMergeRequestLatestMergeRequestDiffIdMigrations, :migration, :sidekiq do
+  let(:projects_table) { table(:projects) }
+  let(:merge_requests_table) { table(:merge_requests) }
+  let(:merge_request_diffs_table) { table(:merge_request_diffs) }
+
+  let(:project) { projects_table.create!(name: 'gitlab', path: 'gitlab-org/gitlab-ce') }
+
+  let!(:merge_request_1) { create_mr!('mr_1', diffs: 1) }
+  let!(:merge_request_2) { create_mr!('mr_2', diffs: 2) }
+  let!(:merge_request_migrated) { create_mr!('merge_request_migrated', diffs: 3) }
+  let!(:merge_request_4) { create_mr!('mr_4', diffs: 3) }
+
+  def create_mr!(name, diffs: 0)
+    merge_request =
+      merge_requests_table.create!(target_project_id: project.id,
+                                   target_branch: 'master',
+                                   source_project_id: project.id,
+                                   source_branch: name,
+                                   title: name)
+
+    diffs.times do
+      merge_request_diffs_table.create!(merge_request_id: merge_request.id)
+    end
+
+    merge_request
+  end
+
+  def diffs_for(merge_request)
+    merge_request_diffs_table.where(merge_request_id: merge_request.id)
+  end
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+
+    diff_id = diffs_for(merge_request_migrated).minimum(:id)
+    merge_request_migrated.update!(latest_merge_request_diff_id: diff_id)
+  end
+
+  it 'correctly schedules background migrations' do
+    Sidekiq::Testing.fake! do
+      Timecop.freeze do
+        migrate!
+
+        expect(described_class::MIGRATION).to be_scheduled_migration(5.minutes, merge_request_1.id, merge_request_1.id)
+        expect(described_class::MIGRATION).to be_scheduled_migration(10.minutes, merge_request_2.id, merge_request_2.id)
+        expect(described_class::MIGRATION).to be_scheduled_migration(15.minutes, merge_request_4.id, merge_request_4.id)
+        expect(BackgroundMigrationWorker.jobs.size).to eq 3
+      end
+    end
+  end
+
+  it 'schedules background migrations' do
+    Sidekiq::Testing.inline! do
+      expect(merge_requests_table.where(latest_merge_request_diff_id: nil).count).to eq 3
+
+      migrate!
+
+      expect(merge_requests_table.where(latest_merge_request_diff_id: nil).count).to eq 0
+    end
+  end
+end