Cleanup after AddPrimaryEmailToEmailsIfUserConfirmed

Finalize the background migration, deal with possible leftovers, and
remove the after_commit logic which is not necessary anymore.

Changelog: changed

db/post_migrate/20211206161271_add_indexes_for_primary_email_cleanup_migration.rb

+# frozen_string_literal: true
+
+class AddIndexesForPrimaryEmailCleanupMigration < Gitlab::Database::Migration[1.0]
+  USERS_INDEX = :index_users_on_id_for_primary_email_migration
+  EMAIL_INDEX = :index_emails_on_email_user_id
+
+  disable_ddl_transaction!
+
+  def up
+    unless index_exists_by_name?(:users, USERS_INDEX)
+
+      disable_statement_timeout do
+        execute <<~SQL
+        CREATE INDEX CONCURRENTLY #{USERS_INDEX}
+        ON users (id) INCLUDE (email, confirmed_at)
+        WHERE confirmed_at IS NOT NULL
+        SQL
+      end
+    end
+
+    add_concurrent_index :emails, [:email, :user_id], name: EMAIL_INDEX
+  end
+
+  def down
+    remove_concurrent_index_by_name :users, USERS_INDEX
+    remove_concurrent_index_by_name :emails, EMAIL_INDEX
+  end
+end

db/post_migrate/20211206162601_cleanup_after_add_primary_email_to_emails_if_user_confirmed.rb

+# frozen_string_literal: true
+
+class CleanupAfterAddPrimaryEmailToEmailsIfUserConfirmed < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  MIGRATION_NAME = 'AddPrimaryEmailToEmailsIfUserConfirmed'
+  BATCH_SIZE = 10_000
+
+  # Stubbed class to access the User table
+  class User < ActiveRecord::Base
+    include ::EachBatch
+
+    self.table_name = 'users'
+    self.inheritance_column = :_type_disabled
+
+    scope :confirmed, -> { where.not(confirmed_at: nil) }
+
+    has_many :emails
+  end
+
+  # Stubbed class to access the Emails table
+  class Email < ActiveRecord::Base
+    self.table_name = 'emails'
+    self.inheritance_column = :_type_disabled
+
+    belongs_to :user
+  end
+
+  def up
+    finalize_background_migration(MIGRATION_NAME)
+
+    # Select confirmed users that do not have their primary email in the emails table,
+    # and create the email record. There should be none if the background migration
+    # completed, but in case there is any leftover, we deal with it synchronously.
+    not_exists_condition = 'NOT EXISTS (SELECT 1 FROM emails WHERE emails.email = users.email AND emails.user_id = users.id)'
+
+    User.confirmed.each_batch(of: BATCH_SIZE) do |user_batch|
+      user_batch.select(:id, :email, :confirmed_at).where(not_exists_condition).each do |user|
+        current_time = Time.now.utc
+
+        begin
+          Email.create(
+            user_id: user.id,
+            email: user.email,
+            confirmed_at: user.confirmed_at,
+            created_at: current_time,
+            updated_at: current_time
+          )
+        rescue StandardError => error
+          Gitlab::AppLogger.error("Could not add primary email #{user.email} to emails for user with ID #{user.id} due to #{error}")
+        end
+      end
+    end
+  end
+
+  def down
+    # Intentionally left blank
+  end
+end

db/post_migrate/20211220064757_drop_temporary_indexes_for_primary_email_migration.rb

+# frozen_string_literal: true
+
+class DropTemporaryIndexesForPrimaryEmailMigration < Gitlab::Database::Migration[1.0]
+  USERS_INDEX = :index_users_on_id_for_primary_email_migration
+  EMAIL_INDEX = :index_emails_on_email_user_id
+
+  disable_ddl_transaction!
+
+  def up
+    remove_concurrent_index_by_name :users, USERS_INDEX
+    remove_concurrent_index_by_name :emails, EMAIL_INDEX
+  end
+
+  def down
+    unless index_exists_by_name?(:users, USERS_INDEX)
+
+      disable_statement_timeout do
+        execute <<~SQL
+        CREATE INDEX CONCURRENTLY #{USERS_INDEX}
+        ON users (id) INCLUDE (email, confirmed_at)
+        WHERE confirmed_at IS NOT NULL
+        SQL
+      end
+    end
+
+    add_concurrent_index :emails, [:email, :user_id], name: EMAIL_INDEX
+  end
+end

db/schema_migrations/20211206161271

+fa4a39c3bea70d31e8144f8830ef0353f22a7a663a891d9043e79f362058fbde
\ No newline at end of file

db/schema_migrations/20211206162601

+529c7ea38bbaa0c29491c2dfdb654a4a6adba93122d9bc23d6632526ff7fdb05
\ No newline at end of file

db/schema_migrations/20211220064757

+34bfe07fff59a415540ca2c5c96b33dc9030c15b2ffbb30cb7deedeb939ae132
\ No newline at end of file

spec/migrations/cleanup_after_add_primary_email_to_emails_if_user_confirmed_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe CleanupAfterAddPrimaryEmailToEmailsIfUserConfirmed, :sidekiq do
+  let(:migration) { described_class.new }
+  let(:users) { table(:users) }
+  let(:emails) { table(:emails) }
+
+  let!(:user_1) { users.create!(name: 'confirmed-user-1', email: 'confirmed-1@example.com', confirmed_at: 3.days.ago, projects_limit: 100) }
+  let!(:user_2) { users.create!(name: 'confirmed-user-2', email: 'confirmed-2@example.com', confirmed_at: 1.day.ago, projects_limit: 100) }
+  let!(:user_3) { users.create!(name: 'confirmed-user-3', email: 'confirmed-3@example.com', confirmed_at: 1.day.ago, projects_limit: 100) }
+  let!(:user_4) { users.create!(name: 'unconfirmed-user', email: 'unconfirmed@example.com', confirmed_at: nil, projects_limit: 100) }
+
+  let!(:email_1) { emails.create!(email: 'confirmed-1@example.com', user_id: user_1.id, confirmed_at: 1.day.ago) }
+  let!(:email_2) { emails.create!(email: 'other_2@example.com', user_id: user_2.id, confirmed_at: 1.day.ago) }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 2)
+  end
+
+  it 'consume any pending background migration job' do
+    expect_next_instance_of(Gitlab::BackgroundMigration::JobCoordinator) do |coordinator|
+      expect(coordinator).to receive(:steal).with('AddPrimaryEmailToEmailsIfUserConfirmed').twice
+    end
+
+    migration.up
+  end
+
+  it 'adds the primary email to emails for leftover confirmed users that do not have their primary email in the emails table', :aggregate_failures do
+    original_email_1_confirmed_at = email_1.reload.confirmed_at
+
+    expect { migration.up }.to change { emails.count }.by(2)
+
+    expect(emails.find_by(user_id: user_2.id, email: 'confirmed-2@example.com').confirmed_at).to eq(user_2.reload.confirmed_at)
+    expect(emails.find_by(user_id: user_3.id, email: 'confirmed-3@example.com').confirmed_at).to eq(user_3.reload.confirmed_at)
+    expect(email_1.reload.confirmed_at).to eq(original_email_1_confirmed_at)
+
+    expect(emails.exists?(user_id: user_4.id)).to be(false)
+  end
+
+  it 'continues in case of errors with one email' do
+    allow(Email).to receive(:create) { raise 'boom!' }
+
+    expect { migration.up }.not_to raise_error
+  end
+end