Commit 2f0668be authored by Grzegorz Bizon's avatar Grzegorz Bizon

Add new migration helpers for data migration in batches

parent 9aae0070
# frozen_string_literal: true # frozen_string_literal: true
class CleanUpPendingBuildsTable < ActiveRecord::Migration[6.0] class CleanUpPendingBuildsTable < ActiveRecord::Migration[6.0]
include ::Gitlab::Database::MigrationHelpers
BATCH_SIZE = 1000 BATCH_SIZE = 1000
disable_ddl_transaction! disable_ddl_transaction!
...@@ -8,7 +10,7 @@ class CleanUpPendingBuildsTable < ActiveRecord::Migration[6.0] ...@@ -8,7 +10,7 @@ class CleanUpPendingBuildsTable < ActiveRecord::Migration[6.0]
def up def up
return unless Gitlab.dev_or_test_env? || Gitlab.com? return unless Gitlab.dev_or_test_env? || Gitlab.com?
each_batch('ci_pending_builds', of: BATCH_SIZE) do |min, max| each_batch_range('ci_pending_builds', of: BATCH_SIZE) do |min, max|
execute <<~SQL execute <<~SQL
DELETE FROM ci_pending_builds DELETE FROM ci_pending_builds
USING ci_builds USING ci_builds
...@@ -23,19 +25,4 @@ class CleanUpPendingBuildsTable < ActiveRecord::Migration[6.0] ...@@ -23,19 +25,4 @@ class CleanUpPendingBuildsTable < ActiveRecord::Migration[6.0]
def down def down
# noop # noop
end end
private
def each_batch(table_name, scope: ->(table) { table.all }, of: 1000)
table = Class.new(ActiveRecord::Base) do
include EachBatch
self.table_name = table_name
self.inheritance_column = :_type_disabled
end
scope.call(table).each_batch(of: of) do |batch|
yield batch.pluck('MIN(id), MAX(id)').first
end
end
end end
...@@ -1591,6 +1591,31 @@ into similar problems in the future (e.g. when new tables are created). ...@@ -1591,6 +1591,31 @@ into similar problems in the future (e.g. when new tables are created).
raise raise
end end
def each_batch(table_name, scope: ->(table) { table.all }, of: 1000)
if transaction_open?
raise <<~MSG.squish
each_batch should not run inside a transaction, you can disable
transactions by calling disable_ddl_transaction! in the body of
your migration class
MSG
end
table = Class.new(ActiveRecord::Base) do
include EachBatch
self.table_name = table_name
self.inheritance_column = :_type_disabled
end
scope.call(table).each_batch(of: of) { |batch| yield batch }
end
def each_batch_range(table_name, scope: ->(table) { table.all }, of: 1000)
each_batch(table_name, scope: scope, of: of) do |batch|
yield batch.pluck('MIN(id), MAX(id)').first
end
end
private private
def validate_check_constraint_name!(constraint_name) def validate_check_constraint_name!(constraint_name)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment