Commit a4c81b64 authored by Yorick Peterse's avatar Yorick Peterse

Merge branch 'fix/gb/improve-updating-column-in-batches-helper' into 'master'

Improve `update_column_in_batches` migration helper

Closes #34064

See merge request !12350
parents 523eaace 019b4d34
...@@ -2,6 +2,8 @@ ...@@ -2,6 +2,8 @@
class SetMissingStageOnCiBuilds < ActiveRecord::Migration class SetMissingStageOnCiBuilds < ActiveRecord::Migration
include Gitlab::Database::MigrationHelpers include Gitlab::Database::MigrationHelpers
disable_ddl_transaction!
def up def up
update_column_in_batches(:ci_builds, :stage, :test) do |table, query| update_column_in_batches(:ci_builds, :stage, :test) do |table, query|
query.where(table[:stage].eq(nil)) query.where(table[:stage].eq(nil))
......
...@@ -5,6 +5,8 @@ class DropAndReaddHasExternalWikiInProjects < ActiveRecord::Migration ...@@ -5,6 +5,8 @@ class DropAndReaddHasExternalWikiInProjects < ActiveRecord::Migration
# Set this constant to true if this migration requires downtime. # Set this constant to true if this migration requires downtime.
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction!
def up def up
update_column_in_batches(:projects, :has_external_wiki, nil) do |table, query| update_column_in_batches(:projects, :has_external_wiki, nil) do |table, query|
query.where(table[:has_external_wiki].not_eq(nil)) query.where(table[:has_external_wiki].not_eq(nil))
......
...@@ -4,6 +4,8 @@ class SetConfidentialIssuesEventsOnWebhooks < ActiveRecord::Migration ...@@ -4,6 +4,8 @@ class SetConfidentialIssuesEventsOnWebhooks < ActiveRecord::Migration
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction!
def up def up
update_column_in_batches(:web_hooks, :confidential_issues_events, true) do |table, query| update_column_in_batches(:web_hooks, :confidential_issues_events, true) do |table, query|
query.where(table[:issues_events].eq(true)) query.where(table[:issues_events].eq(true))
......
...@@ -5,6 +5,8 @@ class AddTypeToLabels < ActiveRecord::Migration ...@@ -5,6 +5,8 @@ class AddTypeToLabels < ActiveRecord::Migration
DOWNTIME = true DOWNTIME = true
DOWNTIME_REASON = 'Labels will not work as expected until this migration is complete.' DOWNTIME_REASON = 'Labels will not work as expected until this migration is complete.'
disable_ddl_transaction!
def change def change
add_column :labels, :type, :string add_column :labels, :type, :string
......
...@@ -4,6 +4,8 @@ class MakeProjectOwnersMasters < ActiveRecord::Migration ...@@ -4,6 +4,8 @@ class MakeProjectOwnersMasters < ActiveRecord::Migration
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction!
def up def up
update_column_in_batches(:members, :access_level, 40) do |table, query| update_column_in_batches(:members, :access_level, 40) do |table, query|
query.where(table[:access_level].eq(50).and(table[:source_type].eq('Project'))) query.where(table[:access_level].eq(50).and(table[:source_type].eq('Project')))
......
...@@ -4,6 +4,8 @@ class RenameSlackAndMattermostNotificationServices < ActiveRecord::Migration ...@@ -4,6 +4,8 @@ class RenameSlackAndMattermostNotificationServices < ActiveRecord::Migration
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction!
def up def up
update_column_in_batches(:services, :type, 'SlackService') do |table, query| update_column_in_batches(:services, :type, 'SlackService') do |table, query|
query.where(table[:type].eq('SlackNotificationService')) query.where(table[:type].eq('SlackNotificationService'))
......
...@@ -4,6 +4,8 @@ class ResetRelativePositionForIssue < ActiveRecord::Migration ...@@ -4,6 +4,8 @@ class ResetRelativePositionForIssue < ActiveRecord::Migration
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction!
def up def up
update_column_in_batches(:issues, :relative_position, nil) do |table, query| update_column_in_batches(:issues, :relative_position, nil) do |table, query|
query.where(table[:relative_position].not_eq(nil)) query.where(table[:relative_position].not_eq(nil))
...@@ -11,5 +13,6 @@ class ResetRelativePositionForIssue < ActiveRecord::Migration ...@@ -11,5 +13,6 @@ class ResetRelativePositionForIssue < ActiveRecord::Migration
end end
def down def down
# noop
end end
end end
...@@ -7,6 +7,8 @@ class UpdateUploadPathsToSystem < ActiveRecord::Migration ...@@ -7,6 +7,8 @@ class UpdateUploadPathsToSystem < ActiveRecord::Migration
DOWNTIME = false DOWNTIME = false
AFFECTED_MODELS = %w(User Project Note Namespace Appearance) AFFECTED_MODELS = %w(User Project Note Namespace Appearance)
disable_ddl_transaction!
def up def up
update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], base_directory, new_upload_dir)) do |_table, query| update_column_in_batches(:uploads, :path, replace_sql(arel_table[:path], base_directory, new_upload_dir)) do |_table, query|
query.where(uploads_to_switch_to_new_path) query.where(uploads_to_switch_to_new_path)
......
...@@ -7,6 +7,8 @@ class MigrateUserProjectView < ActiveRecord::Migration ...@@ -7,6 +7,8 @@ class MigrateUserProjectView < ActiveRecord::Migration
# Set this constant to true if this migration requires downtime. # Set this constant to true if this migration requires downtime.
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction!
def up def up
update_column_in_batches(:users, :project_view, 2) do |table, query| update_column_in_batches(:users, :project_view, 2) do |table, query|
query.where(table[:project_view].eq(0)) query.where(table[:project_view].eq(0))
......
...@@ -3,6 +3,8 @@ class AddHeadPipelineForEachMergeRequest < ActiveRecord::Migration ...@@ -3,6 +3,8 @@ class AddHeadPipelineForEachMergeRequest < ActiveRecord::Migration
DOWNTIME = false DOWNTIME = false
disable_ddl_transaction!
def up def up
disable_statement_timeout disable_statement_timeout
......
...@@ -222,6 +222,12 @@ module Gitlab ...@@ -222,6 +222,12 @@ module Gitlab
# #
# rubocop: disable Metrics/AbcSize # rubocop: disable Metrics/AbcSize
def update_column_in_batches(table, column, value) def update_column_in_batches(table, column, value)
if transaction_open?
raise 'update_column_in_batches can not be run inside a transaction, ' \
'you can disable transactions by calling disable_ddl_transaction! ' \
'in the body of your migration class'
end
table = Arel::Table.new(table) table = Arel::Table.new(table)
count_arel = table.project(Arel.star.count.as('count')) count_arel = table.project(Arel.star.count.as('count'))
......
...@@ -262,39 +262,53 @@ describe Gitlab::Database::MigrationHelpers, lib: true do ...@@ -262,39 +262,53 @@ describe Gitlab::Database::MigrationHelpers, lib: true do
end end
describe '#update_column_in_batches' do describe '#update_column_in_batches' do
before do context 'when running outside of a transaction' do
create_list(:empty_project, 5) before do
end expect(model).to receive(:transaction_open?).and_return(false)
it 'updates all the rows in a table' do create_list(:empty_project, 5)
model.update_column_in_batches(:projects, :import_error, 'foo') end
expect(Project.where(import_error: 'foo').count).to eq(5) it 'updates all the rows in a table' do
end model.update_column_in_batches(:projects, :import_error, 'foo')
it 'updates boolean values correctly' do expect(Project.where(import_error: 'foo').count).to eq(5)
model.update_column_in_batches(:projects, :archived, true) end
expect(Project.where(archived: true).count).to eq(5) it 'updates boolean values correctly' do
end model.update_column_in_batches(:projects, :archived, true)
expect(Project.where(archived: true).count).to eq(5)
end
context 'when a block is supplied' do
it 'yields an Arel table and query object to the supplied block' do
first_id = Project.first.id
context 'when a block is supplied' do model.update_column_in_batches(:projects, :archived, true) do |t, query|
it 'yields an Arel table and query object to the supplied block' do query.where(t[:id].eq(first_id))
first_id = Project.first.id end
model.update_column_in_batches(:projects, :archived, true) do |t, query| expect(Project.where(archived: true).count).to eq(1)
query.where(t[:id].eq(first_id))
end end
end
context 'when the value is Arel.sql (Arel::Nodes::SqlLiteral)' do
it 'updates the value as a SQL expression' do
model.update_column_in_batches(:projects, :star_count, Arel.sql('1+1'))
expect(Project.where(archived: true).count).to eq(1) expect(Project.sum(:star_count)).to eq(2 * Project.count)
end
end end
end end
context 'when the value is Arel.sql (Arel::Nodes::SqlLiteral)' do context 'when running inside the transaction' do
it 'updates the value as a SQL expression' do it 'raises RuntimeError' do
model.update_column_in_batches(:projects, :star_count, Arel.sql('1+1')) expect(model).to receive(:transaction_open?).and_return(true)
expect(Project.sum(:star_count)).to eq(2 * Project.count) expect do
model.update_column_in_batches(:projects, :star_count, Arel.sql('1+1'))
end.to raise_error(RuntimeError)
end end
end end
end end
...@@ -303,7 +317,9 @@ describe Gitlab::Database::MigrationHelpers, lib: true do ...@@ -303,7 +317,9 @@ describe Gitlab::Database::MigrationHelpers, lib: true do
context 'outside of a transaction' do context 'outside of a transaction' do
context 'when a column limit is not set' do context 'when a column limit is not set' do
before do before do
expect(model).to receive(:transaction_open?).and_return(false) expect(model).to receive(:transaction_open?)
.and_return(false)
.at_least(:once)
expect(model).to receive(:transaction).and_yield expect(model).to receive(:transaction).and_yield
...@@ -810,7 +826,11 @@ describe Gitlab::Database::MigrationHelpers, lib: true do ...@@ -810,7 +826,11 @@ describe Gitlab::Database::MigrationHelpers, lib: true do
let!(:user) { create(:user, name: 'Kathy Alice Aliceson') } let!(:user) { create(:user, name: 'Kathy Alice Aliceson') }
it 'replaces the correct part of the string' do it 'replaces the correct part of the string' do
model.update_column_in_batches(:users, :name, model.replace_sql(Arel::Table.new(:users)[:name], 'Alice', 'Eve')) allow(model).to receive(:transaction_open?).and_return(false)
query = model.replace_sql(Arel::Table.new(:users)[:name], 'Alice', 'Eve')
model.update_column_in_batches(:users, :name, query)
expect(user.reload.name).to eq('Kathy Eve Aliceson') expect(user.reload.name).to eq('Kathy Eve Aliceson')
end end
end end
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase do describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameBase, :truncate do
let(:migration) { FakeRenameReservedPathMigrationV1.new } let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) } let(:subject) { described_class.new(['the-path'], migration) }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces do describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameNamespaces, :truncate do
let(:migration) { FakeRenameReservedPathMigrationV1.new } let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) } let(:subject) { described_class.new(['the-path'], migration) }
......
require 'spec_helper' require 'spec_helper'
describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects do describe Gitlab::Database::RenameReservedPathsMigration::V1::RenameProjects, :truncate do
let(:migration) { FakeRenameReservedPathMigrationV1.new } let(:migration) { FakeRenameReservedPathMigrationV1.new }
let(:subject) { described_class.new(['the-path'], migration) } let(:subject) { described_class.new(['the-path'], migration) }
......
...@@ -13,7 +13,7 @@ shared_examples 'renames child namespaces' do |type| ...@@ -13,7 +13,7 @@ shared_examples 'renames child namespaces' do |type|
end end
end end
describe Gitlab::Database::RenameReservedPathsMigration::V1 do describe Gitlab::Database::RenameReservedPathsMigration::V1, :truncate do
let(:subject) { FakeRenameReservedPathMigrationV1.new } let(:subject) { FakeRenameReservedPathMigrationV1.new }
before do before do
......
require 'spec_helper' require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170508170547_add_head_pipeline_for_each_merge_request.rb') require Rails.root.join('db', 'post_migrate', '20170508170547_add_head_pipeline_for_each_merge_request.rb')
describe AddHeadPipelineForEachMergeRequest do describe AddHeadPipelineForEachMergeRequest, :truncate do
let(:migration) { described_class.new } let(:migration) { described_class.new }
let!(:project) { create(:empty_project) } let!(:project) { create(:empty_project) }
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
require 'spec_helper' require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb') require Rails.root.join('db', 'post_migrate', '20170324160416_migrate_user_activities_to_users_last_activity_on.rb')
describe MigrateUserActivitiesToUsersLastActivityOn, :redis do describe MigrateUserActivitiesToUsersLastActivityOn, :redis, :truncate do
let(:migration) { described_class.new } let(:migration) { described_class.new }
let!(:user_active_1) { create(:user) } let!(:user_active_1) { create(:user) }
let!(:user_active_2) { create(:user) } let!(:user_active_2) { create(:user) }
......
...@@ -3,7 +3,7 @@ ...@@ -3,7 +3,7 @@
require 'spec_helper' require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20170406142253_migrate_user_project_view.rb') require Rails.root.join('db', 'post_migrate', '20170406142253_migrate_user_project_view.rb')
describe MigrateUserProjectView do describe MigrateUserProjectView, :truncate do
let(:migration) { described_class.new } let(:migration) { described_class.new }
let!(:user) { create(:user) } let!(:user) { create(:user) }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment