Commit d4ed85ad authored by Kerri Miller's avatar Kerri Miller

Merge branch '30390-duplicate-group-label' into 'master'

Resolve labels with duplicate title and group ID

See merge request gitlab-org/gitlab!37148
parents c952aeb0 191f87b1
---
title: Deduplicate labels with identical title and group
merge_request: 37148
author:
type: fixed
# frozen_string_literal: true
class RemoveDuplicateLabelsFromGroup < ActiveRecord::Migration[6.0]
DOWNTIME = false
CREATE = 1
RENAME = 2
disable_ddl_transaction!
class BackupLabel < ApplicationRecord
include EachBatch
self.table_name = 'backup_labels'
end
class Label < ApplicationRecord
self.table_name = 'labels'
end
class Group < ApplicationRecord
include EachBatch
self.table_name = 'namespaces'
end
BATCH_SIZE = 10_000
def up
# Split to smaller chunks
# Loop rather than background job, every 10,000
# there are ~1,800,000 groups in total (excluding personal namespaces, which can't have labels)
Group.where(type: 'Group').each_batch(of: BATCH_SIZE) do |batch|
range = batch.pluck('MIN(id)', 'MAX(id)').first
transaction do
remove_full_duplicates(*range)
end
transaction do
rename_partial_duplicates(*range)
end
end
end
DOWN_BATCH_SIZE = 1000
def down
BackupLabel.where('project_id IS NULL AND group_id IS NOT NULL').each_batch(of: DOWN_BATCH_SIZE) do |batch|
range = batch.pluck('MIN(id)', 'MAX(id)').first
restore_renamed_labels(*range)
restore_deleted_labels(*range)
end
end
def remove_full_duplicates(start_id, stop_id)
# Fields that are considered duplicate:
# group_id title template description type color
duplicate_labels = ApplicationRecord.connection.execute(<<-SQL.squish)
WITH data AS (
SELECT labels.*,
row_number() OVER (PARTITION BY labels.group_id, labels.title, labels.template, labels.description, labels.type, labels.color ORDER BY labels.id) AS row_number,
#{CREATE} AS restore_action
FROM labels
WHERE labels.group_id BETWEEN #{start_id} AND #{stop_id}
AND NOT EXISTS (SELECT * FROM board_labels WHERE board_labels.label_id = labels.id)
AND NOT EXISTS (SELECT * FROM label_links WHERE label_links.label_id = labels.id)
AND NOT EXISTS (SELECT * FROM label_priorities WHERE label_priorities.label_id = labels.id)
AND NOT EXISTS (SELECT * FROM lists WHERE lists.label_id = labels.id)
AND NOT EXISTS (SELECT * FROM resource_label_events WHERE resource_label_events.label_id = labels.id)
) SELECT * FROM data WHERE row_number > 1;
SQL
if duplicate_labels.any?
# create backup records
BackupLabel.insert_all!(duplicate_labels.map { |label| label.except("row_number") })
Label.unscoped.where(id: duplicate_labels.pluck("id")).delete_all
end
end
def rename_partial_duplicates(start_id, stop_id)
# We need to ensure that the new title (with `_duplicate#{ID}`) doesn't exceed the limit.
# Truncate the original title (if needed) to 245 characters minus the length of the ID
# then add `_duplicate#{ID}`
soft_duplicates = ApplicationRecord.connection.execute(<<-SQL.squish)
WITH data AS (
SELECT
*,
substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text as new_title,
#{RENAME} AS restore_action,
row_number() OVER (PARTITION BY group_id, title ORDER BY id) AS row_number
FROM labels
WHERE group_id BETWEEN #{start_id} AND #{stop_id}
) SELECT * FROM data WHERE row_number > 1;
SQL
if soft_duplicates.any?
# create backup records
BackupLabel.insert_all!(soft_duplicates.map { |label| label.except("row_number") })
ApplicationRecord.connection.execute(<<-SQL.squish)
UPDATE labels SET title = substring(title from 1 for 245 - length(id::text)) || '_duplicate' || id::text
WHERE labels.id IN (#{soft_duplicates.map { |dup| dup["id"] }.join(", ")});
SQL
end
end
def restore_renamed_labels(start_id, stop_id)
# the backup label IDs are not incremental, they are copied directly from the Labels table
ApplicationRecord.connection.execute(<<-SQL.squish)
WITH backups AS (
SELECT id, title
FROM backup_labels
WHERE id BETWEEN #{start_id} AND #{stop_id}
AND restore_action = #{RENAME}
) UPDATE labels SET title = backups.title
FROM backups
WHERE labels.id = backups.id;
SQL
end
def restore_deleted_labels(start_id, stop_id)
ActiveRecord::Base.connection.execute(<<-SQL.squish)
INSERT INTO labels
SELECT id, title, color, group_id, created_at, updated_at, template, description, description_html, type, cached_markdown_version FROM backup_labels
WHERE backup_labels.id BETWEEN #{start_id} AND #{stop_id}
AND backup_labels.project_id IS NULL AND backup_labels.group_id IS NOT NULL
AND backup_labels.restore_action = #{CREATE}
SQL
end
end
# frozen_string_literal: true
class AddUniquenessIndexToLabelTitleAndGroup < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
GROUP_AND_TITLE = [:group_id, :title]
def up
add_concurrent_index :labels, GROUP_AND_TITLE, where: "labels.project_id IS NULL", unique: true, name: "index_labels_on_group_id_and_title_unique"
remove_concurrent_index :labels, GROUP_AND_TITLE, name: "index_labels_on_group_id_and_title"
end
def down
add_concurrent_index :labels, GROUP_AND_TITLE, where: "labels.project_id IS NULL", unique: false, name: "index_labels_on_group_id_and_title"
remove_concurrent_index :labels, GROUP_AND_TITLE, name: "index_labels_on_group_id_and_title_unique"
end
end
9683f55a327b9579b9b0b9484dd11a07b7ea4244b126c46e0144662cb25da6bb
\ No newline at end of file
71cd12e553b3acbb665770fe7478365f1f082e2d278c67b166f41461f689aa5e
\ No newline at end of file
......@@ -21804,7 +21804,7 @@ CREATE UNIQUE INDEX index_label_priorities_on_project_id_and_label_id ON label_p
CREATE UNIQUE INDEX index_labels_on_group_id_and_project_id_and_title ON labels USING btree (group_id, project_id, title);
CREATE INDEX index_labels_on_group_id_and_title_with_null_project_id ON labels USING btree (group_id, title) WHERE (project_id IS NULL);
CREATE UNIQUE INDEX index_labels_on_group_id_and_title_unique ON labels USING btree (group_id, title) WHERE (project_id IS NULL);
CREATE INDEX index_labels_on_project_id ON labels USING btree (project_id);
......
This diff is collapsed.
......@@ -520,14 +520,29 @@ RSpec.describe API::Labels do
expect(json_response['color']).to eq(label1.color)
end
it 'returns 200 if group label already exists' do
create(:group_label, title: label1.name, group: group)
context 'if group label already exists' do
let!(:group_label) { create(:group_label, title: label1.name, group: group) }
expect { put api("/projects/#{project.id}/labels/promote", user), params: { name: label1.name } }
.to change(project.labels, :count).by(-1)
.and change(group.labels, :count).by(0)
it 'returns a status of 200' do
put api("/projects/#{project.id}/labels/promote", user), params: { name: label1.name }
expect(response).to have_gitlab_http_status(:ok)
expect(response).to have_gitlab_http_status(:ok)
end
it 'does not change the group label count' do
expect { put api("/projects/#{project.id}/labels/promote", user), params: { name: label1.name } }
.not_to change(group.labels, :count)
end
it 'does not change the group label max (reuses the same ID)' do
expect { put api("/projects/#{project.id}/labels/promote", user), params: { name: label1.name } }
.not_to change(group.labels, :max)
end
it 'changes the project label count' do
expect { put api("/projects/#{project.id}/labels/promote", user), params: { name: label1.name } }
.to change(project.labels, :count).by(-1)
end
end
it 'returns 403 if guest promotes label' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment