Merge branch 'backfill-routes-namespace-id-for-projects' into 'master'

Backfill routes namespace_id for projects

See merge request gitlab-org/gitlab!81121

db/post_migrate/20220322023800_add_tmp_index_routes_id_for_project_namespaces.rb

+# frozen_string_literal: true
+
+class AddTmpIndexRoutesIdForProjectNamespaces < Gitlab::Database::Migration[1.0]
+  INDEX_NAME = 'tmp_index_for_project_namespace_id_migration_on_routes'
+
+  disable_ddl_transaction!
+
+  def up
+    # Temporary index to be removed in 15.0
+    # https://gitlab.com/gitlab-org/gitlab/-/issues/352353
+    add_concurrent_index :routes, :id, where: "namespace_id IS NULL AND source_type = 'Project'", name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :routes, INDEX_NAME
+  end
+end

db/post_migrate/20220323023800_backfill_namespace_id_for_project_routes.rb

+# frozen_string_literal: true
+
+class BackfillNamespaceIdForProjectRoutes < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  MIGRATION = 'BackfillNamespaceIdForProjectRoute'
+  INTERVAL = 2.minutes
+  BATCH_SIZE = 1_000
+  MAX_BATCH_SIZE = 10_000
+  SUB_BATCH_SIZE = 200
+
+  def up
+    queue_batched_background_migration(
+      MIGRATION,
+      :routes,
+      :id,
+      job_interval: INTERVAL,
+      batch_size: BATCH_SIZE,
+      max_batch_size: MAX_BATCH_SIZE,
+      sub_batch_size: SUB_BATCH_SIZE
+    )
+  end
+
+  def down
+    Gitlab::Database::BackgroundMigration::BatchedMigration
+      .for_configuration(MIGRATION, :routes, :id, [])
+      .delete_all
+  end
+end

db/schema_migrations/20220322023800

+1aefb5950063a060de1ec20b0808a5488b238b36d86120c34ac5a128c212984e
\ No newline at end of file

db/schema_migrations/20220323023800

+1681c19d1f41a05c3dfeded70d128989afb4a81a2e04aacc8879c2c1ab766733
\ No newline at end of file

db/structure.sql

@@ -29667,6 +29667,8 @@ CREATE INDEX tmp_index_for_namespace_id_migration_on_routes ON routes USING btre
 
 CREATE INDEX tmp_index_for_null_project_namespace_id ON projects USING btree (id) WHERE (project_namespace_id IS NULL);
 
+CREATE INDEX tmp_index_for_project_namespace_id_migration_on_routes ON routes USING btree (id) WHERE ((namespace_id IS NULL) AND ((source_type)::text = 'Project'::text));
+
 CREATE INDEX tmp_index_issues_on_issue_type_and_id ON issues USING btree (issue_type, id);
 
 CREATE INDEX tmp_index_members_on_state ON members USING btree (state) WHERE (state = 2);

lib/gitlab/background_migration/backfill_namespace_id_for_project_route.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfills the `routes.namespace_id` column, by setting it to project.project_namespace_id
+    class BackfillNamespaceIdForProjectRoute
+      include Gitlab::Database::DynamicModelHelpers
+
+      def perform(start_id, end_id, batch_table, batch_column, sub_batch_size, pause_ms)
+        parent_batch_relation = relation_scoped_to_range(batch_table, batch_column, start_id, end_id)
+
+        parent_batch_relation.each_batch(column: batch_column, of: sub_batch_size) do |sub_batch|
+          cleanup_gin_index('routes')
+
+          batch_metrics.time_operation(:update_all) do
+            ActiveRecord::Base.connection.execute <<~SQL
+              WITH route_and_ns(route_id, project_namespace_id) AS #{::Gitlab::Database::AsWithMaterialized.materialized_if_supported} (
+                #{sub_batch.to_sql}
+              )
+              UPDATE routes
+              SET namespace_id = route_and_ns.project_namespace_id
+              FROM route_and_ns
+              WHERE id = route_and_ns.route_id
+            SQL
+          end
+
+          pause_ms = [0, pause_ms].max
+          sleep(pause_ms * 0.001)
+        end
+      end
+
+      def batch_metrics
+        @batch_metrics ||= Gitlab::Database::BackgroundMigration::BatchMetrics.new
+      end
+
+      private
+
+      def cleanup_gin_index(table_name)
+        sql = "select indexname::text from pg_indexes where tablename = '#{table_name}' and indexdef ilike '%gin%'"
+        index_names = ActiveRecord::Base.connection.select_values(sql)
+
+        index_names.each do |index_name|
+          ActiveRecord::Base.connection.execute("select gin_clean_pending_list('#{index_name}')")
+        end
+      end
+
+      def relation_scoped_to_range(source_table, source_key_column, start_id, stop_id)
+        define_batchable_model(source_table, connection: ActiveRecord::Base.connection)
+          .joins('INNER JOIN projects ON routes.source_id = projects.id')
+          .where(source_key_column => start_id..stop_id)
+          .where(namespace_id: nil)
+          .where(source_type: 'Project')
+          .where.not(projects: { project_namespace_id: nil })
+          .select("routes.id, projects.project_namespace_id")
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_namespace_id_for_project_route_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillNamespaceIdForProjectRoute do
+  let(:migration) { described_class.new }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:routes) { table(:routes) }
+
+  let(:namespace1) { namespaces.create!(name: 'batchtest1', type: 'Group', path: 'space1') }
+  let(:namespace2) { namespaces.create!(name: 'batchtest2', type: 'Group', parent_id: namespace1.id, path: 'space2') }
+  let(:namespace3) { namespaces.create!(name: 'batchtest3', type: 'Group', parent_id: namespace2.id, path: 'space3') }
+
+  let(:proj_namespace1) { namespaces.create!(name: 'proj1', path: 'proj1', type: 'Project', parent_id: namespace1.id) }
+  let(:proj_namespace2) { namespaces.create!(name: 'proj2', path: 'proj2', type: 'Project', parent_id: namespace2.id) }
+  let(:proj_namespace3) { namespaces.create!(name: 'proj3', path: 'proj3', type: 'Project', parent_id: namespace3.id) }
+  let(:proj_namespace4) { namespaces.create!(name: 'proj4', path: 'proj4', type: 'Project', parent_id: namespace3.id) }
+
+  # rubocop:disable Layout/LineLength
+  let(:proj1) { projects.create!(name: 'proj1', path: 'proj1', namespace_id: namespace1.id, project_namespace_id: proj_namespace1.id) }
+  let(:proj2) { projects.create!(name: 'proj2', path: 'proj2', namespace_id: namespace2.id, project_namespace_id: proj_namespace2.id) }
+  let(:proj3) { projects.create!(name: 'proj3', path: 'proj3', namespace_id: namespace3.id, project_namespace_id: proj_namespace3.id) }
+  let(:proj4) { projects.create!(name: 'proj4', path: 'proj4', namespace_id: namespace3.id, project_namespace_id: proj_namespace4.id) }
+  # rubocop:enable Layout/LineLength
+
+  let!(:namespace_route1) { routes.create!(path: 'space1', source_id: namespace1.id, source_type: 'Namespace') }
+  let!(:namespace_route2) { routes.create!(path: 'space1/space2', source_id: namespace2.id, source_type: 'Namespace') }
+  let!(:namespace_route3) { routes.create!(path: 'space1/space3', source_id: namespace3.id, source_type: 'Namespace') }
+
+  let!(:proj_route1) { routes.create!(path: 'space1/proj1', source_id: proj1.id, source_type: 'Project') }
+  let!(:proj_route2) { routes.create!(path: 'space1/space2/proj2', source_id: proj2.id, source_type: 'Project') }
+  let!(:proj_route3) { routes.create!(path: 'space1/space3/proj3', source_id: proj3.id, source_type: 'Project') }
+  let!(:proj_route4) { routes.create!(path: 'space1/space3/proj4', source_id: proj4.id, source_type: 'Project') }
+
+  subject(:perform_migration) { migration.perform(proj_route1.id, proj_route4.id, :routes, :id, 2, 0) }
+
+  it 'backfills namespace_id for the selected records', :aggregate_failures do
+    perform_migration
+
+    expected_namespaces = [proj_namespace1.id, proj_namespace2.id, proj_namespace3.id, proj_namespace4.id]
+
+    expected_projects = [proj_route1.id, proj_route2.id, proj_route3.id, proj_route4.id]
+    expect(routes.where.not(namespace_id: nil).pluck(:id)).to match_array(expected_projects)
+    expect(routes.where.not(namespace_id: nil).pluck(:namespace_id)).to match_array(expected_namespaces)
+  end
+
+  it 'tracks timings of queries' do
+    expect(migration.batch_metrics.timings).to be_empty
+
+    expect { perform_migration }.to change { migration.batch_metrics.timings }
+  end
+end

spec/migrations/backfill_namespace_id_for_project_routes_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe BackfillNamespaceIdForProjectRoutes, :migration do
+  let(:migration) { described_class::MIGRATION }
+
+  describe '#up' do
+    it 'schedules background jobs for each batch of group members' do
+      migrate!
+
+      expect(migration).to have_scheduled_batched_migration(
+        table_name: :routes,
+        column_name: :id,
+        interval: described_class::INTERVAL
+      )
+    end
+  end
+
+  describe '#down' do
+    it 'deletes all batched migration records' do
+      migrate!
+      schema_migrate_down!
+
+      expect(migration).not_to have_scheduled_batched_migration
+    end
+  end
+end