Merge branch 'ab/async-index-creation' into 'master'

Async index creation

See merge request gitlab-org/gitlab!66478

db/migrate/20210720140841_create_postgres_async_indexes_table.rb

+# frozen_string_literal: true
+
+class CreatePostgresAsyncIndexesTable < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  def change
+    create_table_with_constraints :postgres_async_indexes do |t|
+      t.timestamps_with_timezone null: false
+
+      t.text :name, null: false
+      t.text :definition, null: false
+      t.text :table_name, null: false
+
+      t.text_limit :name, 63
+      t.text_limit :definition, 2048
+      t.text_limit :table_name, 63
+
+      t.index :name, unique: true
+    end
+  end
+end

db/schema_migrations/20210720140841

+1ef66bdf4a1c61d9a1e0e632d8728f86769ac727d43971e897284272e9f53581
\ No newline at end of file

db/structure.sql

@@ -16549,6 +16549,27 @@ CREATE SEQUENCE pool_repositories_id_seq
 
 ALTER SEQUENCE pool_repositories_id_seq OWNED BY pool_repositories.id;
 
+CREATE TABLE postgres_async_indexes (
+    id bigint NOT NULL,
+    created_at timestamp with time zone NOT NULL,
+    updated_at timestamp with time zone NOT NULL,
+    name text NOT NULL,
+    definition text NOT NULL,
+    table_name text NOT NULL,
+    CONSTRAINT check_083b21157b CHECK ((char_length(definition) <= 2048)),
+    CONSTRAINT check_b732c6cd1d CHECK ((char_length(name) <= 63)),
+    CONSTRAINT check_e64ff4359e CHECK ((char_length(table_name) <= 63))
+);
+
+CREATE SEQUENCE postgres_async_indexes_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE postgres_async_indexes_id_seq OWNED BY postgres_async_indexes.id;
+
 CREATE VIEW postgres_foreign_keys AS
  SELECT pg_constraint.oid,
     pg_constraint.conname AS name,
@@ -20513,6 +20534,8 @@ ALTER TABLE ONLY plans ALTER COLUMN id SET DEFAULT nextval('plans_id_seq'::regcl
 
 ALTER TABLE ONLY pool_repositories ALTER COLUMN id SET DEFAULT nextval('pool_repositories_id_seq'::regclass);
 
+ALTER TABLE ONLY postgres_async_indexes ALTER COLUMN id SET DEFAULT nextval('postgres_async_indexes_id_seq'::regclass);
+
 ALTER TABLE ONLY postgres_reindex_actions ALTER COLUMN id SET DEFAULT nextval('postgres_reindex_actions_id_seq'::regclass);
 
 ALTER TABLE ONLY product_analytics_events_experimental ALTER COLUMN id SET DEFAULT nextval('product_analytics_events_experimental_id_seq'::regclass);
@@ -22077,6 +22100,9 @@ ALTER TABLE ONLY plans
 ALTER TABLE ONLY pool_repositories
     ADD CONSTRAINT pool_repositories_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY postgres_async_indexes
+    ADD CONSTRAINT postgres_async_indexes_pkey PRIMARY KEY (id);
+
 ALTER TABLE ONLY postgres_reindex_actions
     ADD CONSTRAINT postgres_reindex_actions_pkey PRIMARY KEY (id);
 
@@ -24635,6 +24661,8 @@ CREATE INDEX index_pool_repositories_on_shard_id ON pool_repositories USING btre
 
 CREATE UNIQUE INDEX index_pool_repositories_on_source_project_id_and_shard_id ON pool_repositories USING btree (source_project_id, shard_id);
 
+CREATE UNIQUE INDEX index_postgres_async_indexes_on_name ON postgres_async_indexes USING btree (name);
+
 CREATE INDEX index_postgres_reindex_actions_on_index_identifier ON postgres_reindex_actions USING btree (index_identifier);
 
 CREATE UNIQUE INDEX index_programming_languages_on_name ON programming_languages USING btree (name);

doc/development/adding_database_indexes.md

@@ -226,3 +226,88 @@ def down
   remove_concurrent_index_by_name :projects, INDEX_NAME
 end
 ```
+
+## Create indexes asynchronously
+
+For very large tables, index creation can be a challenge to manage.
+While `add_concurrent_index` creates indexes in a way that does not block
+normal traffic, it can still be problematic when index creation runs for
+many hours. Necessary database operations like `autovacuum` cannot run, and
+on GitLab.com, the deployment process is blocked waiting for index
+creation to finish.
+
+To limit impact on GitLab.com, a process exists to create indexes
+asynchronously during weekend hours. Due to generally lower levels of
+traffic and lack of regular deployments, this process allows the
+creation of indexes to proceed with a lower level of risk. The below
+sections describe the steps required to use these features:
+
+1. [Schedule the index to be created](#schedule-the-index-to-be-created).
+1. [Verify the MR was deployed and the index exists in production](#verify-the-mr-was-deployed-and-the-index-exists-in-production).
+1. [Add a migration to create the index synchronously](#add-a-migration-to-create-the-index-synchronously).
+
+### Schedule the index to be created
+
+Create an MR with a post-deployment migration which prepares the index
+for asynchronous creation. An example of creating an index using
+the asynchronous index helpers can be seen in the block below. This migration
+enters the index name and definition into the `postgres_async_indexes`
+table. The process that runs on weekends pulls indexes from this
+table and attempt to create them.
+
+```ruby
+# in db/post_migrate/
+
+INDEX_NAME = 'index_ci_builds_on_some_column'
+
+def up
+  prepare_async_index :ci_builds, :some_column, name: INDEX_NAME
+end
+
+def down
+  unprepare_async_index :ci_builds, :some_column, name: INDEX_NAME
+end
+```
+
+### Verify the MR was deployed and the index exists in production
+
+You can verify if the MR was deployed to GitLab.com by executing
+`/chatops run auto_deploy status <merge_sha>`. To verify existence of
+the index, you can:
+
+- Use a meta-command in #database-lab, such as: `\di <index_name>`
+- Ask someone in #database to check if the index exists
+- With proper access, you can also verify directly on production or in a
+production clone
+
+### Add a migration to create the index synchronously
+
+After the index is verified to exist on the production database, create a second
+merge request that adds the index synchronously. The synchronous
+migration results in a no-op on GitLab.com, but you should still add the
+migration as expected for other installations. The below block
+demonstrates how to create the second migration for the previous
+asynchronous example.
+
+WARNING:
+The responsibility lies on the individual writing the migrations to verify
+the index exists in production before merging a second migration that
+adds the index using `add_concurrent_index`. If the second migration is
+deployed and the index has not yet been created, the index is created
+synchronously when the second migration executes.
+
+```ruby
+# in db/post_migrate/
+
+INDEX_NAME = 'index_ci_builds_on_some_column'
+
+disable_ddl_transaction!
+
+def up
+  add_concurrent_index :ci_builds, :some_column, name: INDEX_NAME
+end
+
+def down
+  remove_concurrent_index_by_name :ci_builds, INDEX_NAME
+end
+```

lib/gitlab/database/async_indexes.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      DEFAULT_INDEXES_PER_INVOCATION = 2
+
+      def self.create_pending_indexes!(how_many: DEFAULT_INDEXES_PER_INVOCATION)
+        PostgresAsyncIndex.order(:id).limit(how_many).each do |async_index|
+          IndexCreator.new(async_index).perform
+        end
+      end
+    end
+  end
+end

lib/gitlab/database/async_indexes/index_creator.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      class IndexCreator
+        include ExclusiveLeaseGuard
+
+        TIMEOUT_PER_ACTION = 1.day
+        STATEMENT_TIMEOUT = 9.hours
+
+        def initialize(async_index)
+          @async_index = async_index
+        end
+
+        def perform
+          try_obtain_lease do
+            if index_exists?
+              log_index_info('Skipping index creation as the index exists')
+            else
+              log_index_info('Creating async index')
+
+              set_statement_timeout do
+                connection.execute(async_index.definition)
+              end
+            end
+
+            async_index.destroy
+          end
+        end
+
+        private
+
+        attr_reader :async_index
+
+        def index_exists?
+          connection.indexes(async_index.table_name).any? { |index| index.name == async_index.name }
+        end
+
+        def connection
+          @connection ||= ApplicationRecord.connection
+        end
+
+        def lease_timeout
+          TIMEOUT_PER_ACTION
+        end
+
+        def set_statement_timeout
+          connection.execute("SET statement_timeout TO '%ds'" % STATEMENT_TIMEOUT)
+          yield
+        ensure
+          connection.execute('RESET statement_timeout')
+        end
+
+        def log_index_info(message)
+          Gitlab::AppLogger.info(message: message, table_name: async_index.table_name, index_name: async_index.name)
+        end
+      end
+    end
+  end
+end

lib/gitlab/database/async_indexes/migration_helpers.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      module MigrationHelpers
+        def unprepare_async_index(table_name, column_name, **options)
+          return unless async_index_creation_available?
+
+          index_name = options[:name] || index_name(table_name, column_name)
+
+          raise 'Specifying index name is mandatory - specify name: argument' unless index_name
+
+          unprepare_async_index_by_name(table_name, index_name)
+        end
+
+        def unprepare_async_index_by_name(table_name, index_name, **options)
+          return unless async_index_creation_available?
+
+          PostgresAsyncIndex.find_by(name: index_name).try do |async_index|
+            async_index.destroy
+          end
+        end
+
+        # Prepares an index for asynchronous creation.
+        #
+        # Stores the index information in the postgres_async_indexes table to be created later. The
+        # index will be always be created CONCURRENTLY, so that option does not need to be given.
+        # If an existing asynchronous definition exists with the same name, the existing entry will be
+        # updated with the new definition.
+        #
+        # If the requested index has already been created, it is not stored in the table for
+        # asynchronous creation.
+        def prepare_async_index(table_name, column_name, **options)
+          return unless async_index_creation_available?
+
+          index_name = options[:name] || index_name(table_name, column_name)
+
+          raise 'Specifying index name is mandatory - specify name: argument' unless index_name
+
+          options = options.merge({ algorithm: :concurrently })
+
+          if index_exists?(table_name, column_name, **options)
+            Gitlab::AppLogger.warn(
+              message: 'Index not prepared because it already exists',
+              table_name: table_name,
+              index_name: index_name)
+
+            return
+          end
+
+          index, algorithm, if_not_exists = add_index_options(table_name, column_name, **options)
+
+          create_index = ActiveRecord::ConnectionAdapters::CreateIndexDefinition.new(index, algorithm, if_not_exists)
+          schema_creation = ActiveRecord::ConnectionAdapters::PostgreSQL::SchemaCreation.new(ApplicationRecord.connection)
+          definition = schema_creation.accept(create_index)
+
+          async_index = PostgresAsyncIndex.safe_find_or_create_by!(name: index_name) do |rec|
+            rec.table_name = table_name
+            rec.definition = definition
+          end
+
+          Gitlab::AppLogger.info(
+            message: 'Prepared index for async creation',
+            table_name: async_index.table_name,
+            index_name: async_index.name)
+
+          async_index
+        end
+
+        private
+
+        def async_index_creation_available?
+          ApplicationRecord.connection.table_exists?(:postgres_async_indexes) &&
+            Feature.enabled?(:database_async_index_creation, type: :ops)
+        end
+      end
+    end
+  end
+end

lib/gitlab/database/async_indexes/postgres_async_index.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module Database
+    module AsyncIndexes
+      class PostgresAsyncIndex < ApplicationRecord
+        self.table_name = 'postgres_async_indexes'
+
+        MAX_IDENTIFIER_LENGTH = Gitlab::Database::MigrationHelpers::MAX_IDENTIFIER_NAME_LENGTH
+        MAX_DEFINITION_LENGTH = 2048
+
+        validates :name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+        validates :table_name, presence: true, length: { maximum: MAX_IDENTIFIER_LENGTH }
+        validates :definition, presence: true, length: { maximum: MAX_DEFINITION_LENGTH }
+
+        def to_s
+          definition
+        end
+      end
+    end
+  end
+end

lib/gitlab/database/migration_helpers.rb

@@ -6,6 +6,7 @@ module Gitlab
       include Migrations::BackgroundMigrationHelpers
       include DynamicModelHelpers
       include RenameTableHelpers
+      include AsyncIndexes::MigrationHelpers
 
       # https://www.postgresql.org/docs/current/sql-syntax-lexical.html#SQL-SYNTAX-IDENTIFIERS
       MAX_IDENTIFIER_NAME_LENGTH = 63
@@ -152,6 +153,9 @@ module Gitlab
         disable_statement_timeout do
           add_index(table_name, column_name, **options)
         end
+
+        # We created this index. Now let's remove the queuing entry for async creation in case it's still there.
+        unprepare_async_index(table_name, column_name, **options)
       end
 
       # Removes an existed index, concurrently
@@ -178,6 +182,9 @@ module Gitlab
         disable_statement_timeout do
           remove_index(table_name, **options.merge({ column: column_name }))
         end
+
+        # We removed this index. Now let's make sure it's not queued for async creation.
+        unprepare_async_index(table_name, column_name, **options)
       end
 
       # Removes an existing index, concurrently
@@ -208,6 +215,9 @@ module Gitlab
         disable_statement_timeout do
           remove_index(table_name, **options.merge({ name: index_name }))
         end
+
+        # We removed this index. Now let's make sure it's not queued for async creation.
+        unprepare_async_index_by_name(table_name, index_name, **options)
       end
 
       # Adds a foreign key with only minimal locking on the tables involved.

lib/tasks/gitlab/db.rake

@@ -176,6 +176,9 @@ namespace :gitlab do
       # Cleanup leftover temporary indexes from previous, possibly aborted runs (if any)
       Gitlab::Database::Reindexing.cleanup_leftovers!
 
+      # Hack: Before we do actual reindexing work, create async indexes
+      Gitlab::Database::AsyncIndexes.create_pending_indexes! if Feature.enabled?(:database_async_index_creation, type: :ops)
+
       Gitlab::Database::Reindexing.perform(indexes)
     rescue StandardError => e
       Gitlab::AppLogger.error(e)

spec/factories/gitlab/database/async_indexes/postgres_async_index.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :postgres_async_index, class: 'Gitlab::Database::AsyncIndexes::PostgresAsyncIndex' do
+    sequence(:name) { |n| "users_id_#{n}" }
+    definition { "CREATE INDEX #{name} ON #{table_name} (id)" }
+    table_name { "users" }
+  end
+end

spec/lib/gitlab/database/async_indexes/index_creator_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Database::AsyncIndexes::IndexCreator do
+  describe '#perform' do
+    subject { described_class.new(async_index) }
+
+    let(:async_index) { create(:postgres_async_index) }
+
+    let(:index_model) { Gitlab::Database::AsyncIndexes::PostgresAsyncIndex }
+
+    let(:connection) { ApplicationRecord.connection }
+
+    context 'when the index already exists' do
+      before do
+        connection.execute(async_index.definition)
+      end
+
+      it 'skips index creation' do
+        expect(connection).not_to receive(:execute).with(/CREATE INDEX/)
+
+        subject.perform
+      end
+    end
+
+    it 'creates the index while controlling statement timeout' do
+      allow(connection).to receive(:execute).and_call_original
+      expect(connection).to receive(:execute).with("SET statement_timeout TO '32400s'").ordered.and_call_original
+      expect(connection).to receive(:execute).with(async_index.definition).ordered.and_call_original
+      expect(connection).to receive(:execute).with("RESET statement_timeout").ordered.and_call_original
+
+      subject.perform
+    end
+
+    it 'removes the index preparation record from postgres_async_indexes' do
+      expect(async_index).to receive(:destroy).and_call_original
+
+      expect { subject.perform }.to change { index_model.count }.by(-1)
+    end
+
+    it 'skips logic if not able to acquire exclusive lease' do
+      expect(subject).to receive(:try_obtain_lease).and_return(false)
+      expect(connection).not_to receive(:execute).with(/CREATE INDEX/)
+      expect(async_index).not_to receive(:destroy)
+
+      expect { subject.perform }.not_to change { index_model.count }
+    end
+  end
+end

spec/lib/gitlab/database/async_indexes/migration_helpers_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Database::AsyncIndexes::MigrationHelpers do
+  let(:migration) { ActiveRecord::Migration.new.extend(described_class) }
+  let(:index_model) { Gitlab::Database::AsyncIndexes::PostgresAsyncIndex }
+  let(:connection) { ApplicationRecord.connection }
+  let(:table_name) { '_test_async_indexes' }
+  let(:index_name) { "index_#{table_name}_on_id" }
+
+  before do
+    allow(migration).to receive(:puts)
+  end
+
+  describe '#unprepare_async_index' do
+    let!(:async_index) { create(:postgres_async_index, name: index_name) }
+
+    context 'when the flag is enabled' do
+      before do
+        stub_feature_flags(database_async_index_creation: true)
+      end
+
+      it 'destroys the record' do
+        expect do
+          migration.unprepare_async_index(table_name, 'id')
+        end.to change { index_model.where(name: index_name).count }.by(-1)
+      end
+
+      context 'when an explicit name is given' do
+        let(:index_name) { 'my_test_async_index' }
+
+        it 'destroys the record' do
+          expect do
+            migration.unprepare_async_index(table_name, 'id', name: index_name)
+          end.to change { index_model.where(name: index_name).count }.by(-1)
+        end
+      end
+
+      context 'when the async index table does not exist' do
+        it 'does not raise an error' do
+          connection.drop_table(:postgres_async_indexes)
+
+          expect(index_model).not_to receive(:find_by)
+
+          expect { migration.unprepare_async_index(table_name, 'id') }.not_to raise_error
+        end
+      end
+    end
+
+    context 'when the feature flag is disabled' do
+      it 'does not destroy the record' do
+        stub_feature_flags(database_async_index_creation: false)
+
+        expect do
+          migration.unprepare_async_index(table_name, 'id')
+        end.not_to change { index_model.where(name: index_name).count }
+      end
+    end
+  end
+
+  describe '#unprepare_async_index_by_name' do
+    let(:index_name) { "index_#{table_name}_on_id" }
+    let!(:async_index) { create(:postgres_async_index, name: index_name) }
+
+    context 'when the flag is enabled' do
+      before do
+        stub_feature_flags(database_async_index_creation: true)
+      end
+
+      it 'destroys the record' do
+        expect do
+          migration.unprepare_async_index_by_name(table_name, index_name)
+        end.to change { index_model.where(name: index_name).count }.by(-1)
+      end
+
+      context 'when the async index table does not exist' do
+        it 'does not raise an error' do
+          connection.drop_table(:postgres_async_indexes)
+
+          expect(index_model).not_to receive(:find_by)
+
+          expect { migration.unprepare_async_index_by_name(table_name, index_name) }.not_to raise_error
+        end
+      end
+    end
+
+    context 'when the feature flag is disabled' do
+      it 'does not destroy the record' do
+        stub_feature_flags(database_async_index_creation: false)
+
+        expect do
+          migration.unprepare_async_index_by_name(table_name, index_name)
+        end.not_to change { index_model.where(name: index_name).count }
+      end
+    end
+  end
+
+  describe '#prepare_async_index' do
+    before do
+      connection.create_table(table_name)
+    end
+
+    context 'when the feature flag is enabled' do
+      before do
+        stub_feature_flags(database_async_index_creation: true)
+      end
+
+      it 'creates the record for the async index' do
+        expect do
+          migration.prepare_async_index(table_name, 'id')
+        end.to change { index_model.where(name: index_name).count }.by(1)
+
+        record = index_model.find_by(name: index_name)
+
+        expect(record.table_name).to eq(table_name)
+        expect(record.definition).to match(/CREATE INDEX CONCURRENTLY "#{index_name}"/)
+      end
+
+      context 'when an explicit name is given' do
+        let(:index_name) { 'my_async_index_name' }
+
+        it 'creates the record with the given name' do
+          expect do
+            migration.prepare_async_index(table_name, 'id', name: index_name)
+          end.to change { index_model.where(name: index_name).count }.by(1)
+
+          record = index_model.find_by(name: index_name)
+
+          expect(record.table_name).to eq(table_name)
+          expect(record.definition).to match(/CREATE INDEX CONCURRENTLY "#{index_name}"/)
+        end
+      end
+
+      context 'when the index already exists' do
+        it 'does not create the record' do
+          connection.add_index(table_name, 'id', name: index_name)
+
+          expect do
+            migration.prepare_async_index(table_name, 'id')
+          end.not_to change { index_model.where(name: index_name).count }
+        end
+      end
+
+      context 'when the record already exists' do
+        it 'does attempt to create the record' do
+          create(:postgres_async_index, table_name: table_name, name: index_name)
+
+          expect do
+            migration.prepare_async_index(table_name, 'id')
+          end.not_to change { index_model.where(name: index_name).count }
+        end
+      end
+
+      context 'when the async index table does not exist' do
+        it 'does not raise an error' do
+          connection.drop_table(:postgres_async_indexes)
+
+          expect(index_model).not_to receive(:safe_find_or_create_by!)
+
+          expect { migration.prepare_async_index(table_name, 'id') }.not_to raise_error
+        end
+      end
+    end
+
+    context 'when the feature flag is disabled' do
+      it 'does not create the record' do
+        stub_feature_flags(database_async_index_creation: false)
+
+        expect do
+          migration.prepare_async_index(table_name, 'id')
+        end.not_to change { index_model.where(name: index_name).count }
+      end
+    end
+  end
+end

spec/lib/gitlab/database/async_indexes/postgres_async_index_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Database::AsyncIndexes::PostgresAsyncIndex, type: :model do
+  describe 'validations' do
+    let(:identifier_limit) { described_class::MAX_IDENTIFIER_LENGTH }
+    let(:definition_limit) { described_class::MAX_DEFINITION_LENGTH }
+
+    it { is_expected.to validate_presence_of(:name) }
+    it { is_expected.to validate_length_of(:name).is_at_most(identifier_limit) }
+    it { is_expected.to validate_presence_of(:table_name) }
+    it { is_expected.to validate_length_of(:table_name).is_at_most(identifier_limit) }
+    it { is_expected.to validate_presence_of(:definition) }
+    it { is_expected.to validate_length_of(:definition).is_at_most(definition_limit) }
+  end
+end

spec/lib/gitlab/database/async_indexes_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::Database::AsyncIndexes do
+  describe '.create_pending_indexes!' do
+    subject { described_class.create_pending_indexes! }
+
+    before do
+      create_list(:postgres_async_index, 4)
+    end
+
+    it 'takes 2 pending indexes and creates those' do
+      Gitlab::Database::AsyncIndexes::PostgresAsyncIndex.order(:id).limit(2).each do |index|
+        creator = double('index creator')
+        expect(Gitlab::Database::AsyncIndexes::IndexCreator).to receive(:new).with(index).and_return(creator)
+        expect(creator).to receive(:perform)
+      end
+
+      subject
+    end
+  end
+end

spec/lib/gitlab/database/migration_helpers_spec.rb

@@ -278,6 +278,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
 
         model.add_concurrent_index(:users, :foo, unique: true)
       end
+
+      it 'unprepares the async index creation' do
+        expect(model).to receive(:add_index)
+          .with(:users, :foo, algorithm: :concurrently)
+
+        expect(model).to receive(:unprepare_async_index)
+          .with(:users, :foo, algorithm: :concurrently)
+
+        model.add_concurrent_index(:users, :foo)
+      end
     end
 
     context 'inside a transaction' do
@@ -314,6 +324,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
           model.remove_concurrent_index(:users, :foo, unique: true)
         end
 
+        it 'unprepares the async index creation' do
+          expect(model).to receive(:remove_index)
+            .with(:users, { algorithm: :concurrently, column: :foo })
+
+          expect(model).to receive(:unprepare_async_index)
+            .with(:users, :foo, { algorithm: :concurrently })
+
+          model.remove_concurrent_index(:users, :foo)
+        end
+
         describe 'by index name' do
           before do
             allow(model).to receive(:index_exists_by_name?).with(:users, "index_x_by_y").and_return(true)
@@ -345,6 +365,16 @@ RSpec.describe Gitlab::Database::MigrationHelpers do
               model.remove_concurrent_index_by_name(:users, wrong_key: "index_x_by_y")
             end.to raise_error 'remove_concurrent_index_by_name must get an index name as the second argument'
           end
+
+          it 'unprepares the async index creation' do
+            expect(model).to receive(:remove_index)
+              .with(:users, { algorithm: :concurrently, name: "index_x_by_y" })
+
+            expect(model).to receive(:unprepare_async_index_by_name)
+              .with(:users, "index_x_by_y", { algorithm: :concurrently })
+
+            model.remove_concurrent_index_by_name(:users, "index_x_by_y")
+          end
         end
       end
     end

spec/tasks/gitlab/db_rake_spec.rb

@@ -207,6 +207,27 @@ RSpec.describe 'gitlab:db namespace rake task', :silence_stdout do
       run_rake_task('gitlab:db:reindex')
     end
 
+    context 'when async index creation is enabled' do
+      it 'executes async index creation prior to any reindexing actions' do
+        stub_feature_flags(database_async_index_creation: true)
+
+        expect(Gitlab::Database::AsyncIndexes).to receive(:create_pending_indexes!).ordered
+        expect(Gitlab::Database::Reindexing).to receive(:perform).ordered
+
+        run_rake_task('gitlab:db:reindex')
+      end
+    end
+
+    context 'when async index creation is disabled' do
+      it 'does not execute async index creation' do
+        stub_feature_flags(database_async_index_creation: false)
+
+        expect(Gitlab::Database::AsyncIndexes).not_to receive(:create_pending_indexes!)
+
+        run_rake_task('gitlab:db:reindex')
+      end
+    end
+
     context 'when no index_name is given' do
       it 'uses all candidate indexes' do
         expect(Gitlab::Database::PostgresIndex).to receive(:reindexing_support).and_return(indexes)