Skip to content

G
gitlab-ce
Commit d13e8efd authored by Mayra Cabrera's avatar Mayra Cabrera
Browse files
Options

Merge branch 'fix-partitioning-code-inconsistencies' into 'master' 

Address inconsistent names in partitioning helpers

See merge request gitlab-org/gitlab!35466
parents d493441d cc5287b4
Hide whitespace changes
Inline Side-by-side
Showing with 89 additions and 89 deletions
lib/gitlab/database/partitioning_migration_helpers/table_management_helpers.rb
View file @ d13e8efd
...@@ -51,12 +51,12 @@ module Gitlab ...@@ -51,12 +51,12 @@ module Gitlab
partition_column = find_column_definition(table_name, column_name) partition_column = find_column_definition(table_name, column_name)
raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil? raise "partition column #{column_name} does not exist on #{table_name}" if partition_column.nil?
new_table_name = partitioned_table_name(table_name) partitioned_table_name = make_partitioned_table_name(table_name)
create_range_partitioned_copy(new_table_name, table_name, partition_column, primary_key) create_range_partitioned_copy(table_name, partitioned_table_name, partition_column, primary_key)
create_daterange_partitions(new_table_name, partition_column.name, min_date, max_date) create_daterange_partitions(partitioned_table_name, partition_column.name, min_date, max_date)
create_trigger_to_sync_tables(table_name, new_table_name, primary_key) create_trigger_to_sync_tables(table_name, partitioned_table_name, primary_key)
enqueue_background_migration(table_name, new_table_name, primary_key) enqueue_background_migration(table_name, partitioned_table_name, primary_key)
end end
# Clean up a partitioned copy of an existing table. This deletes the partitioned table and all partitions. # Clean up a partitioned copy of an existing table. This deletes the partitioned table and all partitions.
...@@ -70,15 +70,15 @@ module Gitlab ...@@ -70,15 +70,15 @@ module Gitlab
assert_not_in_transaction_block(scope: ERROR_SCOPE) assert_not_in_transaction_block(scope: ERROR_SCOPE)
with_lock_retries do with_lock_retries do
trigger_name = sync_trigger_name(table_name) trigger_name = make_sync_trigger_name(table_name)
drop_trigger(table_name, trigger_name) drop_trigger(table_name, trigger_name)
end end
function_name = sync_function_name(table_name) function_name = make_sync_function_name(table_name)
drop_function(function_name) drop_function(function_name)
part_table_name = partitioned_table_name(table_name) partitioned_table_name = make_partitioned_table_name(table_name)
drop_table(part_table_name) drop_table(partitioned_table_name)
end end
def create_hash_partitions(table_name, number_of_partitions) def create_hash_partitions(table_name, number_of_partitions)
...@@ -107,15 +107,15 @@ module Gitlab ...@@ -107,15 +107,15 @@ module Gitlab
"for more information please contact the database team" "for more information please contact the database team"
end end
def partitioned_table_name(table) def make_partitioned_table_name(table)
tmp_table_name("#{table}_part") tmp_table_name("#{table}_part")
end end
def sync_function_name(table) def make_sync_function_name(table)
object_name(table, 'table_sync_function') object_name(table, 'table_sync_function')
end end
def sync_trigger_name(table) def make_sync_trigger_name(table)
object_name(table, 'table_sync_trigger') object_name(table, 'table_sync_trigger')
end end
...@@ -123,11 +123,11 @@ module Gitlab ...@@ -123,11 +123,11 @@ module Gitlab
connection.columns(table).find { |c| c.name == column.to_s } connection.columns(table).find { |c| c.name == column.to_s }
end end
def create_range_partitioned_copy(table_name, template_table_name, partition_column, primary_key) def create_range_partitioned_copy(source_table_name, partitioned_table_name, partition_column, primary_key)
if table_exists?(table_name) if table_exists?(partitioned_table_name)
# rubocop:disable Gitlab/RailsLogger # rubocop:disable Gitlab/RailsLogger
Rails.logger.warn "Partitioned table not created because it already exists" \ Rails.logger.warn "Partitioned table not created because it already exists" \
" (this may be due to an aborted migration or similar): table_name: #{table_name} " " (this may be due to an aborted migration or similar): table_name: #{partitioned_table_name} "
# rubocop:enable Gitlab/RailsLogger # rubocop:enable Gitlab/RailsLogger
return return
end end
...@@ -135,20 +135,20 @@ module Gitlab ...@@ -135,20 +135,20 @@ module Gitlab
tmp_column_name = object_name(partition_column.name, 'partition_key') tmp_column_name = object_name(partition_column.name, 'partition_key')
transaction do transaction do
execute(<<~SQL) execute(<<~SQL)
CREATE TABLE #{table_name} ( CREATE TABLE #{partitioned_table_name} (
LIKE #{template_table_name} INCLUDING ALL EXCLUDING INDEXES, LIKE #{source_table_name} INCLUDING ALL EXCLUDING INDEXES,
#{tmp_column_name} #{partition_column.sql_type} NOT NULL, #{tmp_column_name} #{partition_column.sql_type} NOT NULL,
PRIMARY KEY (#{[primary_key, tmp_column_name].join(", ")}) PRIMARY KEY (#{[primary_key, tmp_column_name].join(", ")})
) PARTITION BY RANGE (#{tmp_column_name}) ) PARTITION BY RANGE (#{tmp_column_name})
SQL SQL
remove_column(table_name, partition_column.name) remove_column(partitioned_table_name, partition_column.name)
rename_column(table_name, tmp_column_name, partition_column.name) rename_column(partitioned_table_name, tmp_column_name, partition_column.name)
change_column_default(table_name, primary_key, nil) change_column_default(partitioned_table_name, primary_key, nil)
if column_of_type?(table_name, primary_key, :integer) if column_of_type?(partitioned_table_name, primary_key, :integer)
# Default to int8 primary keys to prevent overflow # Default to int8 primary keys to prevent overflow
change_column(table_name, primary_key, :bigint) change_column(partitioned_table_name, primary_key, :bigint)
end end
end end
end end
...@@ -191,19 +191,19 @@ module Gitlab ...@@ -191,19 +191,19 @@ module Gitlab
create_range_partition(partition_name, table_name, lower_bound, upper_bound) create_range_partition(partition_name, table_name, lower_bound, upper_bound)
end end
def create_trigger_to_sync_tables(source_table, target_table, unique_key) def create_trigger_to_sync_tables(source_table_name, partitioned_table_name, unique_key)
function_name = sync_function_name(source_table) function_name = make_sync_function_name(source_table_name)
trigger_name = sync_trigger_name(source_table) trigger_name = make_sync_trigger_name(source_table_name)
with_lock_retries do with_lock_retries do
create_sync_function(function_name, target_table, unique_key) create_sync_function(function_name, partitioned_table_name, unique_key)
create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table} table") create_comment('FUNCTION', function_name, "Partitioning migration: table sync for #{source_table_name} table")
create_sync_trigger(source_table, trigger_name, function_name) create_sync_trigger(source_table_name, trigger_name, function_name)
end end
end end
def create_sync_function(name, target_table, unique_key) def create_sync_function(name, partitioned_table_name, unique_key)
if function_exists?(name) if function_exists?(name)
# rubocop:disable Gitlab/RailsLogger # rubocop:disable Gitlab/RailsLogger
Rails.logger.warn "Partitioning sync function not created because it already exists" \ Rails.logger.warn "Partitioning sync function not created because it already exists" \
...@@ -213,20 +213,20 @@ module Gitlab ...@@ -213,20 +213,20 @@ module Gitlab
end end
delimiter = ",\n " delimiter = ",\n "
column_names = connection.columns(target_table).map(&:name) column_names = connection.columns(partitioned_table_name).map(&:name)
set_statements = build_set_statements(column_names, unique_key) set_statements = build_set_statements(column_names, unique_key)
insert_values = column_names.map { |name| "NEW.#{name}" } insert_values = column_names.map { |name| "NEW.#{name}" }
create_trigger_function(name, replace: false) do create_trigger_function(name, replace: false) do
<<~SQL <<~SQL
IF (TG_OP = 'DELETE') THEN IF (TG_OP = 'DELETE') THEN
DELETE FROM #{target_table} where #{unique_key} = OLD.#{unique_key}; DELETE FROM #{partitioned_table_name} where #{unique_key} = OLD.#{unique_key};
ELSIF (TG_OP = 'UPDATE') THEN ELSIF (TG_OP = 'UPDATE') THEN
UPDATE #{target_table} UPDATE #{partitioned_table_name}
SET #{set_statements.join(delimiter)} SET #{set_statements.join(delimiter)}
WHERE #{target_table}.#{unique_key} = NEW.#{unique_key}; WHERE #{partitioned_table_name}.#{unique_key} = NEW.#{unique_key};
ELSIF (TG_OP = 'INSERT') THEN ELSIF (TG_OP = 'INSERT') THEN
INSERT INTO #{target_table} (#{column_names.join(delimiter)}) INSERT INTO #{partitioned_table_name} (#{column_names.join(delimiter)})
VALUES (#{insert_values.join(delimiter)}); VALUES (#{insert_values.join(delimiter)});
END IF; END IF;
RETURN NULL; RETURN NULL;
...@@ -250,15 +250,15 @@ module Gitlab ...@@ -250,15 +250,15 @@ module Gitlab
create_trigger(table_name, trigger_name, function_name, fires: 'AFTER INSERT OR UPDATE OR DELETE') create_trigger(table_name, trigger_name, function_name, fires: 'AFTER INSERT OR UPDATE OR DELETE')
end end
def enqueue_background_migration(source_table, partitioned_table, source_key) def enqueue_background_migration(source_table_name, partitioned_table_name, source_key)
model_class = define_batchable_model(source_table) source_model = define_batchable_model(source_table_name)
queue_background_migration_jobs_by_range_at_intervals( queue_background_migration_jobs_by_range_at_intervals(
model_class, source_model,
MIGRATION_CLASS_NAME, MIGRATION_CLASS_NAME,
BATCH_INTERVAL, BATCH_INTERVAL,
batch_size: BATCH_SIZE, batch_size: BATCH_SIZE,
other_job_arguments: [source_table.to_s, partitioned_table, source_key]) other_job_arguments: [source_table_name.to_s, partitioned_table_name, source_key])
end end
end end
end end
......
spec/lib/gitlab/database/partitioning_migration_helpers/table_management_helpers_spec.rb
View file @ d13e8efd
...@@ -11,7 +11,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -11,7 +11,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
let_it_be(:connection) { ActiveRecord::Base.connection } let_it_be(:connection) { ActiveRecord::Base.connection }
let(:template_table) { :audit_events } let(:source_table) { :audit_events }
let(:partitioned_table) { '_test_migration_partitioned_table' } let(:partitioned_table) { '_test_migration_partitioned_table' }
let(:function_name) { '_test_migration_function_name' } let(:function_name) { '_test_migration_function_name' }
let(:trigger_name) { '_test_migration_trigger_name' } let(:trigger_name) { '_test_migration_trigger_name' }
...@@ -22,9 +22,9 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -22,9 +22,9 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
before do before do
allow(migration).to receive(:puts) allow(migration).to receive(:puts)
allow(migration).to receive(:transaction_open?).and_return(false) allow(migration).to receive(:transaction_open?).and_return(false)
allow(migration).to receive(:partitioned_table_name).and_return(partitioned_table) allow(migration).to receive(:make_partitioned_table_name).and_return(partitioned_table)
allow(migration).to receive(:sync_function_name).and_return(function_name) allow(migration).to receive(:make_sync_function_name).and_return(function_name)
allow(migration).to receive(:sync_trigger_name).and_return(trigger_name) allow(migration).to receive(:make_sync_trigger_name).and_return(trigger_name)
allow(migration).to receive(:assert_table_is_allowed) allow(migration).to receive(:assert_table_is_allowed)
end end
...@@ -38,14 +38,14 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -38,14 +38,14 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
context 'when the table is not allowed' do context 'when the table is not allowed' do
let(:template_table) { :this_table_is_not_allowed } let(:source_table) { :this_table_is_not_allowed }
it 'raises an error' do it 'raises an error' do
expect(migration).to receive(:assert_table_is_allowed).with(template_table).and_call_original expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original
expect do expect do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
end.to raise_error(/#{template_table} is not allowed for use/) end.to raise_error(/#{source_table} is not allowed for use/)
end end
end end
...@@ -54,7 +54,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -54,7 +54,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
expect(migration).to receive(:transaction_open?).and_return(true) expect(migration).to receive(:transaction_open?).and_return(true)
expect do expect do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
end.to raise_error(/can not be run inside a transaction/) end.to raise_error(/can not be run inside a transaction/)
end end
end end
...@@ -64,7 +64,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -64,7 +64,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
it 'raises an error' do it 'raises an error' do
expect do expect do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
end.to raise_error(/max_date #{max_date} must be greater than min_date #{min_date}/) end.to raise_error(/max_date #{max_date} must be greater than min_date #{min_date}/)
end end
end end
...@@ -74,24 +74,24 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -74,24 +74,24 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
it 'raises an error' do it 'raises an error' do
expect do expect do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
end.to raise_error(/max_date #{max_date} must be greater than min_date #{min_date}/) end.to raise_error(/max_date #{max_date} must be greater than min_date #{min_date}/)
end end
end end
context 'when the given table does not have a primary key' do context 'when the given table does not have a primary key' do
let(:template_table) { :_partitioning_migration_helper_test_table } let(:source_table) { :_partitioning_migration_helper_test_table }
let(:partition_column) { :some_field } let(:partition_column) { :some_field }
it 'raises an error' do it 'raises an error' do
migration.create_table template_table, id: false do |t| migration.create_table source_table, id: false do |t|
t.integer :id t.integer :id
t.datetime partition_column t.datetime partition_column
end end
expect do expect do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
end.to raise_error(/primary key not defined for #{template_table}/) end.to raise_error(/primary key not defined for #{source_table}/)
end end
end end
...@@ -100,14 +100,14 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -100,14 +100,14 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
it 'raises an error' do it 'raises an error' do
expect do expect do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
end.to raise_error(/partition column #{partition_column} does not exist/) end.to raise_error(/partition column #{partition_column} does not exist/)
end end
end end
describe 'constructing the partitioned table' do describe 'constructing the partitioned table' do
it 'creates a table partitioned by the proper column' do it 'creates a table partitioned by the proper column' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
expect(connection.table_exists?(partitioned_table)).to be(true) expect(connection.table_exists?(partitioned_table)).to be(true)
expect(connection.primary_key(partitioned_table)).to eq(new_primary_key) expect(connection.primary_key(partitioned_table)).to eq(new_primary_key)
...@@ -116,7 +116,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -116,7 +116,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
it 'changes the primary key datatype to bigint' do it 'changes the primary key datatype to bigint' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key } pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key }
...@@ -131,13 +131,13 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -131,13 +131,13 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
end end
let(:template_table) { :another_example } let(:source_table) { :another_example }
let(:old_primary_key) { 'identifier' } let(:old_primary_key) { 'identifier' }
it 'does not change the primary key datatype' do it 'does not change the primary key datatype' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
original_pk_column = connection.columns(template_table).find { |c| c.name == old_primary_key } original_pk_column = connection.columns(source_table).find { |c| c.name == old_primary_key }
pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key } pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key }
expect(pk_column).not_to be_nil expect(pk_column).not_to be_nil
...@@ -146,7 +146,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -146,7 +146,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
it 'removes the default from the primary key column' do it 'removes the default from the primary key column' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key } pk_column = connection.columns(partitioned_table).find { |c| c.name == old_primary_key }
...@@ -154,16 +154,16 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -154,16 +154,16 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
it 'creates the partitioned table with the same non-key columns' do it 'creates the partitioned table with the same non-key columns' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
copied_columns = filter_columns_by_name(connection.columns(partitioned_table), new_primary_key) copied_columns = filter_columns_by_name(connection.columns(partitioned_table), new_primary_key)
original_columns = filter_columns_by_name(connection.columns(template_table), new_primary_key) original_columns = filter_columns_by_name(connection.columns(source_table), new_primary_key)
expect(copied_columns).to match_array(original_columns) expect(copied_columns).to match_array(original_columns)
end end
it 'creates a partition spanning over each month in the range given' do it 'creates a partition spanning over each month in the range given' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
expect_range_partitions_for(partitioned_table, { expect_range_partitions_for(partitioned_table, {
'000000' => ['MINVALUE', "'2019-12-01 00:00:00'"], '000000' => ['MINVALUE', "'2019-12-01 00:00:00'"],
...@@ -175,7 +175,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -175,7 +175,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
context 'when min_date is not given' do context 'when min_date is not given' do
let(:template_table) { :todos } let(:source_table) { :todos }
context 'with records present already' do context 'with records present already' do
before do before do
...@@ -183,7 +183,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -183,7 +183,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
it 'creates a partition spanning over each month from the first record' do it 'creates a partition spanning over each month from the first record' do
migration.partition_table_by_date template_table, partition_column, max_date: max_date migration.partition_table_by_date source_table, partition_column, max_date: max_date
expect_range_partitions_for(partitioned_table, { expect_range_partitions_for(partitioned_table, {
'000000' => ['MINVALUE', "'2019-11-01 00:00:00'"], '000000' => ['MINVALUE', "'2019-11-01 00:00:00'"],
...@@ -198,7 +198,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -198,7 +198,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
context 'without data' do context 'without data' do
it 'creates the catchall partition plus two actual partition' do it 'creates the catchall partition plus two actual partition' do
migration.partition_table_by_date template_table, partition_column, max_date: max_date migration.partition_table_by_date source_table, partition_column, max_date: max_date
expect_range_partitions_for(partitioned_table, { expect_range_partitions_for(partitioned_table, {
'000000' => ['MINVALUE', "'2020-02-01 00:00:00'"], '000000' => ['MINVALUE', "'2020-02-01 00:00:00'"],
...@@ -214,7 +214,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -214,7 +214,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
today = Date.new(2020, 5, 8) today = Date.new(2020, 5, 8)
Timecop.freeze(today) do Timecop.freeze(today) do
migration.partition_table_by_date template_table, partition_column, min_date: min_date migration.partition_table_by_date source_table, partition_column, min_date: min_date
expect_range_partitions_for(partitioned_table, { expect_range_partitions_for(partitioned_table, {
'000000' => ['MINVALUE', "'2019-12-01 00:00:00'"], '000000' => ['MINVALUE', "'2019-12-01 00:00:00'"],
...@@ -234,7 +234,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -234,7 +234,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
it 'creates partitions for the current and next month' do it 'creates partitions for the current and next month' do
current_date = Date.new(2020, 05, 22) current_date = Date.new(2020, 05, 22)
Timecop.freeze(current_date.to_time) do Timecop.freeze(current_date.to_time) do
migration.partition_table_by_date template_table, partition_column migration.partition_table_by_date source_table, partition_column
expect_range_partitions_for(partitioned_table, { expect_range_partitions_for(partitioned_table, {
'000000' => ['MINVALUE', "'2020-05-01 00:00:00'"], '000000' => ['MINVALUE', "'2020-05-01 00:00:00'"],
...@@ -247,7 +247,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -247,7 +247,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
describe 'keeping data in sync with the partitioned table' do describe 'keeping data in sync with the partitioned table' do
let(:template_table) { :todos } let(:source_table) { :todos }
let(:model) { Class.new(ActiveRecord::Base) } let(:model) { Class.new(ActiveRecord::Base) }
let(:timestamp) { Time.utc(2019, 12, 1, 12).round } let(:timestamp) { Time.utc(2019, 12, 1, 12).round }
...@@ -258,16 +258,16 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -258,16 +258,16 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
it 'creates a trigger function on the original table' do it 'creates a trigger function on the original table' do
expect_function_not_to_exist(function_name) expect_function_not_to_exist(function_name)
expect_trigger_not_to_exist(template_table, trigger_name) expect_trigger_not_to_exist(source_table, trigger_name)
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
expect_function_to_exist(function_name) expect_function_to_exist(function_name)
expect_valid_function_trigger(template_table, trigger_name, function_name, after: %w[delete insert update]) expect_valid_function_trigger(source_table, trigger_name, function_name, after: %w[delete insert update])
end end
it 'syncs inserts to the partitioned tables' do it 'syncs inserts to the partitioned tables' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
expect(model.count).to eq(0) expect(model.count).to eq(0)
...@@ -280,7 +280,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -280,7 +280,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
it 'syncs updates to the partitioned tables' do it 'syncs updates to the partitioned tables' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
first_todo = create(:todo, :pending, commit_id: nil, created_at: timestamp, updated_at: timestamp) first_todo = create(:todo, :pending, commit_id: nil, created_at: timestamp, updated_at: timestamp)
second_todo = create(:todo, created_at: timestamp, updated_at: timestamp) second_todo = create(:todo, created_at: timestamp, updated_at: timestamp)
...@@ -301,7 +301,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -301,7 +301,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
it 'syncs deletes to the partitioned tables' do it 'syncs deletes to the partitioned tables' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
first_todo = create(:todo, created_at: timestamp, updated_at: timestamp) first_todo = create(:todo, created_at: timestamp, updated_at: timestamp)
second_todo = create(:todo, created_at: timestamp, updated_at: timestamp) second_todo = create(:todo, created_at: timestamp, updated_at: timestamp)
...@@ -317,7 +317,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -317,7 +317,7 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
describe 'copying historic data to the partitioned table' do describe 'copying historic data to the partitioned table' do
let(:template_table) { 'todos' } let(:source_table) { 'todos' }
let(:migration_class) { '::Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' } let(:migration_class) { '::Gitlab::Database::PartitioningMigrationHelpers::BackfillPartitionedTable' }
let(:sub_batch_size) { described_class::SUB_BATCH_SIZE } let(:sub_batch_size) { described_class::SUB_BATCH_SIZE }
let(:pause_seconds) { described_class::PAUSE_SECONDS } let(:pause_seconds) { described_class::PAUSE_SECONDS }
...@@ -333,14 +333,14 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -333,14 +333,14 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
it 'enqueues jobs to copy each batch of data' do it 'enqueues jobs to copy each batch of data' do
Sidekiq::Testing.fake! do Sidekiq::Testing.fake! do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
expect(BackgroundMigrationWorker.jobs.size).to eq(2) expect(BackgroundMigrationWorker.jobs.size).to eq(2)
first_job_arguments = [first_id, second_id, template_table, partitioned_table, 'id'] first_job_arguments = [first_id, second_id, source_table, partitioned_table, 'id']
expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([migration_class, first_job_arguments]) expect(BackgroundMigrationWorker.jobs[0]['args']).to eq([migration_class, first_job_arguments])
second_job_arguments = [third_id, third_id, template_table, partitioned_table, 'id'] second_job_arguments = [third_id, third_id, source_table, partitioned_table, 'id']
expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([migration_class, second_job_arguments]) expect(BackgroundMigrationWorker.jobs[1]['args']).to eq([migration_class, second_job_arguments])
end end
end end
...@@ -353,37 +353,37 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe ...@@ -353,37 +353,37 @@ RSpec.describe Gitlab::Database::PartitioningMigrationHelpers::TableManagementHe
end end
context 'when the table is not allowed' do context 'when the table is not allowed' do
let(:template_table) { :this_table_is_not_allowed } let(:source_table) { :this_table_is_not_allowed }
it 'raises an error' do it 'raises an error' do
expect(migration).to receive(:assert_table_is_allowed).with(template_table).and_call_original expect(migration).to receive(:assert_table_is_allowed).with(source_table).and_call_original
expect do expect do
migration.drop_partitioned_table_for template_table migration.drop_partitioned_table_for source_table
end.to raise_error(/#{template_table} is not allowed for use/) end.to raise_error(/#{source_table} is not allowed for use/)
end end
end end
it 'drops the trigger syncing to the partitioned table' do it 'drops the trigger syncing to the partitioned table' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
expect_function_to_exist(function_name) expect_function_to_exist(function_name)
expect_valid_function_trigger(template_table, trigger_name, function_name, after: %w[delete insert update]) expect_valid_function_trigger(source_table, trigger_name, function_name, after: %w[delete insert update])
migration.drop_partitioned_table_for template_table migration.drop_partitioned_table_for source_table
expect_function_not_to_exist(function_name) expect_function_not_to_exist(function_name)
expect_trigger_not_to_exist(template_table, trigger_name) expect_trigger_not_to_exist(source_table, trigger_name)
end end
it 'drops the partitioned copy and all partitions' do it 'drops the partitioned copy and all partitions' do
migration.partition_table_by_date template_table, partition_column, min_date: min_date, max_date: max_date migration.partition_table_by_date source_table, partition_column, min_date: min_date, max_date: max_date
expected_tables.each do |table| expected_tables.each do |table|
expect(connection.table_exists?(table)).to be(true) expect(connection.table_exists?(table)).to be(true)
end end
migration.drop_partitioned_table_for template_table migration.drop_partitioned_table_for source_table
expected_tables.each do |table| expected_tables.each do |table|
expect(connection.table_exists?(table)).to be(false) expect(connection.table_exists?(table)).to be(false)
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7