Finalize conversion to bigint for ci_builds_metadata

Changelog: other

db/migrate/20210708011425_rename_ci_builds_metadata_foreign_key.rb

+# frozen_string_literal: true
+
+class RenameCiBuildsMetadataForeignKey < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  TABLE_NAME = 'ci_builds_metadata'
+  OLD_PREFIX = 'fk_rails_'
+
+  def up
+    with_lock_retries(raise_on_exhaustion: true) do
+      rename_constraint(
+        TABLE_NAME,
+        concurrent_foreign_key_name(TABLE_NAME, :build_id, prefix: 'fk_rails_'),
+        concurrent_foreign_key_name(TABLE_NAME, :build_id)
+      )
+    end
+  end
+
+  def down
+    with_lock_retries(raise_on_exhaustion: true) do
+      rename_constraint(
+        TABLE_NAME,
+        concurrent_foreign_key_name(TABLE_NAME, :build_id),
+        concurrent_foreign_key_name(TABLE_NAME, :build_id, prefix: 'fk_rails_')
+      )
+    end
+  end
+end

db/post_migrate/20210708011426_finalize_ci_builds_metadata_bigint_conversion.rb

+# frozen_string_literal: true
+
+class FinalizeCiBuildsMetadataBigintConversion < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  TABLE_NAME = 'ci_builds_metadata'
+
+  def up
+    # TODO: Do these together or in separate migrations? Should the FK swap be done with ci_builds PK swap?
+    ensure_batched_background_migration_is_finished(
+      job_class_name: 'CopyColumnUsingBackgroundMigrationJob',
+      table_name: TABLE_NAME,
+      column_name: 'id',
+      job_arguments: [["id"], ["id_convert_to_bigint"]]
+    )
+
+    ensure_batched_background_migration_is_finished(
+      job_class_name: 'CopyColumnUsingBackgroundMigrationJob',
+      table_name: TABLE_NAME,
+      column_name: 'build_id',
+      job_arguments: [["build_id"], ["build_id_convert_to_bigint"]]
+    )
+
+    swap
+  end
+
+  def down
+    swap
+  end
+
+  private
+
+  def swap
+    # Indexes were pre-created on gitlab.com to avoid slowing down deployments
+    #
+    # rubocop:disable Migration/PreventIndexCreation
+    add_concurrent_index TABLE_NAME, :id_convert_to_bigint, unique: true, name: 'index_ci_builds_metadata_on_id_convert_to_bigint'
+    add_concurrent_index TABLE_NAME, :build_id_convert_to_bigint, where: 'has_exposed_artifacts IS TRUE', name: 'index_ci_builds_metadata_on_build_id_int8_and_exposed_artifacts'
+    create_covering_index TABLE_NAME, 'index_ci_builds_metadata_on_build_id_int8_where_interruptible'
+    add_concurrent_index TABLE_NAME, :build_id_convert_to_bigint, unique: true, name: 'index_ci_builds_metadata_on_build_id_convert_to_bigint'
+    # rubocop:enable Migration/PreventIndexCreation
+
+    # TODO: Also do this in advance on gitlab.com
+    add_concurrent_foreign_key TABLE_NAME, :ci_builds, column: :build_id_convert_to_bigint, on_delete: :cascade,
+                               reverse_lock_order: true
+
+    with_lock_retries(raise_on_exhaustion: true) do
+      # TODO: Locking two high traffic tables could cause downtime
+      execute "LOCK TABLE ci_builds, #{TABLE_NAME} IN ACCESS EXCLUSIVE MODE"
+
+      # rubocop:disable Migration/WithLockRetriesDisallowedMethod
+      swap_column :id
+      swap_column :build_id
+      # rubocop:enable Migration/WithLockRetriesDisallowedMethod
+
+      # We need to update the trigger function in order to make PostgreSQL to
+      # regenerate the execution plan for it. This is to avoid type mismatch errors like
+      # "type of parameter 15 (bigint) does not match that when preparing the plan (integer)"
+      execute "ALTER FUNCTION #{quote_table_name(Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:id, :id_convert_to_bigint))} RESET ALL"
+      execute "ALTER FUNCTION #{quote_table_name(Gitlab::Database::UnidirectionalCopyTrigger.on_table(TABLE_NAME).name(:build_id, :build_id_convert_to_bigint))} RESET ALL"
+
+      # Swap defaults for PK
+      execute "ALTER SEQUENCE ci_builds_metadata_id_seq OWNED BY #{TABLE_NAME}.id"
+      change_column_default TABLE_NAME, :id, -> { "nextval('ci_builds_metadata_id_seq'::regclass)" }
+      change_column_default TABLE_NAME, :id_convert_to_bigint, 0
+
+      # Swap defaults for FK
+      change_column_default TABLE_NAME, :build_id, nil
+      change_column_default TABLE_NAME, :build_id_convert_to_bigint, 0
+
+      # Swap PK constraint
+      execute "ALTER TABLE #{TABLE_NAME} DROP CONSTRAINT ci_builds_metadata_pkey CASCADE"
+      rename_index TABLE_NAME, 'index_ci_builds_metadata_on_id_convert_to_bigint', 'ci_builds_metadata_pkey'
+      execute "ALTER TABLE #{TABLE_NAME} ADD CONSTRAINT ci_builds_metadata_pkey PRIMARY KEY USING INDEX ci_builds_metadata_pkey"
+
+      # Rename the rest of the indexes (we already hold an exclusive lock, so no need to use DROP INDEX CONCURRENTLY here)
+      # rubocop:disable Migration/WithLockRetriesDisallowedMethod
+      swap_index 'index_ci_builds_metadata_on_build_id', 'index_ci_builds_metadata_on_build_id_convert_to_bigint'
+      swap_index 'index_ci_builds_metadata_on_build_id_and_has_exposed_artifacts', 'index_ci_builds_metadata_on_build_id_int8_and_exposed_artifacts'
+      swap_index 'index_ci_builds_metadata_on_build_id_and_id_and_interruptible', 'index_ci_builds_metadata_on_build_id_int8_where_interruptible'
+      # rubocop:enable Migration/WithLockRetriesDisallowedMethod
+
+      # Swap FK constraint
+      remove_foreign_key TABLE_NAME, name: concurrent_foreign_key_name(TABLE_NAME, :build_id)
+      rename_constraint(
+        TABLE_NAME,
+        concurrent_foreign_key_name(TABLE_NAME, :build_id_convert_to_bigint),
+        concurrent_foreign_key_name(TABLE_NAME, :build_id)
+      )
+    end
+  end
+
+  def swap_index(old, new)
+    execute "DROP INDEX #{old}"
+    rename_index TABLE_NAME, new, old
+  end
+
+  def swap_column(name)
+    temp_name = "#{name}_tmp"
+    execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(name)} TO #{quote_column_name(temp_name)}"
+    execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(:"#{name}_convert_to_bigint")} TO #{quote_column_name(name)}"
+    execute "ALTER TABLE #{quote_table_name(TABLE_NAME)} RENAME COLUMN #{quote_column_name(temp_name)} TO #{quote_column_name(:"#{name}_convert_to_bigint")}"
+  end
+
+  def create_covering_index(table, name)
+    return if index_exists_by_name?(table, name)
+
+    disable_statement_timeout do
+      execute <<~SQL
+        CREATE INDEX CONCURRENTLY #{name}
+        ON #{table} (build_id_convert_to_bigint) INCLUDE (id_convert_to_bigint)
+        WHERE interruptible = true
+      SQL
+    end
+  end
+end

db/schema_migrations/20210708011425

+e3f4424daaba173f607dbae7c84b4f6070126d262e7e9808c6a90e64648e10ed
\ No newline at end of file

db/schema_migrations/20210708011426

+b48556968cbff2e3aff65236b313ed8a626af4a08b1cad06723b74a99b678895
\ No newline at end of file

db/structure.sql

@@ -11360,8 +11360,8 @@ CREATE SEQUENCE ci_builds_id_seq
 ALTER SEQUENCE ci_builds_id_seq OWNED BY ci_builds.id;
 
 CREATE TABLE ci_builds_metadata (
-    id integer NOT NULL,
-    build_id integer NOT NULL,
+    id_convert_to_bigint integer DEFAULT 0 NOT NULL,
+    build_id_convert_to_bigint integer DEFAULT 0 NOT NULL,
     project_id integer NOT NULL,
     timeout integer,
     timeout_source integer DEFAULT 1 NOT NULL,
@@ -11372,8 +11372,8 @@ CREATE TABLE ci_builds_metadata (
     environment_auto_stop_in character varying(255),
     expanded_environment_name character varying(255),
     secrets jsonb DEFAULT '{}'::jsonb NOT NULL,
-    build_id_convert_to_bigint bigint DEFAULT 0 NOT NULL,
-    id_convert_to_bigint bigint DEFAULT 0 NOT NULL
+    build_id bigint NOT NULL,
+    id bigint NOT NULL
 );
 
 CREATE SEQUENCE ci_builds_metadata_id_seq
@@ -28020,6 +28020,9 @@ ALTER TABLE ONLY ci_resources
 ALTER TABLE ONLY ci_sources_pipelines
     ADD CONSTRAINT fk_e1bad85861 FOREIGN KEY (pipeline_id) REFERENCES ci_pipelines(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY ci_builds_metadata
+    ADD CONSTRAINT fk_e20479742e FOREIGN KEY (build_id) REFERENCES ci_builds(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY gitlab_subscriptions
     ADD CONSTRAINT fk_e2595d00a1 FOREIGN KEY (namespace_id) REFERENCES namespaces(id) ON DELETE CASCADE;
 
@@ -29553,9 +29556,6 @@ ALTER TABLE ONLY packages_packages
 ALTER TABLE ONLY cluster_platforms_kubernetes
     ADD CONSTRAINT fk_rails_e1e2cf841a FOREIGN KEY (cluster_id) REFERENCES clusters(id) ON DELETE CASCADE;
 
-ALTER TABLE ONLY ci_builds_metadata
-    ADD CONSTRAINT fk_rails_e20479742e FOREIGN KEY (build_id) REFERENCES ci_builds(id) ON DELETE CASCADE;
-
 ALTER TABLE ONLY vulnerability_finding_evidences
     ADD CONSTRAINT fk_rails_e3205a0c65 FOREIGN KEY (vulnerability_occurrence_id) REFERENCES vulnerability_occurrences(id) ON DELETE CASCADE;