Skip to content

G
gitlab-ce
Commit 0a5c6c33 authored by Kyle Wiebers's avatar Kyle Wiebers
Browse files
Options

Revert "Merge branch '21801-migrate-epic-mentions-to-db-table' into 'master'" 

This reverts merge request !22333
parent 8240d01a
Hide whitespace changes
Inline Side-by-side
Showing with 4 additions and 539 deletions
changelogs/unreleased/21801-migrate-epic-and-epic-notes-mentions-to-epic-user-mentions-table.yml deleted 100644 → 0
View file @ 8240d01a
---
title: Migrate epic, epic notes mentions to respective DB table
merge_request: 22333
author:
type: changed
db/post_migrate/20191115115043_migrate_epic_mentions_to_db.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
class MigrateEpicMentionsToDb < ActiveRecord::Migration[5.2]
DOWNTIME = false
disable_ddl_transaction!
DELAY = 2.minutes.to_i
BATCH_SIZE = 10000
MIGRATION = 'UserMentions::CreateResourceUserMention'
JOIN = "LEFT JOIN epic_user_mentions on epics.id = epic_user_mentions.epic_id"
QUERY_CONDITIONS = "(description like '%@%' OR title like '%@%') AND epic_user_mentions.epic_id is null"
class Epic < ActiveRecord::Base
include EachBatch
self.table_name = 'epics'
end
def up
return unless Gitlab.ee?
Epic
.joins(JOIN)
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(epics.id)'), Arel.sql('MAX(epics.id)')).first
BackgroundMigrationWorker.perform_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, false, *range])
end
end
def down
# no-op
end
end
db/post_migrate/20191115115522_migrate_epic_notes_mentions_to_db.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
class MigrateEpicNotesMentionsToDb < ActiveRecord::Migration[5.2]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
disable_ddl_transaction!
DELAY = 2.minutes.to_i
BATCH_SIZE = 10000
MIGRATION = 'UserMentions::CreateResourceUserMention'
INDEX_NAME = 'epic_mentions_temp_index'
INDEX_CONDITION = "note LIKE '%@%'::text AND notes.noteable_type = 'Epic'"
QUERY_CONDITIONS = "#{INDEX_CONDITION} AND epic_user_mentions.epic_id IS NULL"
JOIN = 'LEFT JOIN epic_user_mentions ON notes.id = epic_user_mentions.note_id'
class Note < ActiveRecord::Base
include EachBatch
self.table_name = 'notes'
end
def up
return unless Gitlab.ee?
# create temporary index for notes with mentions, may take well over 1h
add_concurrent_index(:notes, :id, where: INDEX_CONDITION, name: INDEX_NAME)
Note
.joins(JOIN)
.where(QUERY_CONDITIONS)
.each_batch(of: BATCH_SIZE) do |batch, index|
range = batch.pluck(Arel.sql('MIN(notes.id)'), Arel.sql('MAX(notes.id)')).first
BackgroundMigrationWorker.perform_in(index * DELAY, MIGRATION, ['Epic', JOIN, QUERY_CONDITIONS, true, *range])
end
end
def down
# no-op
# temporary index is to be dropped in a different migration in an upcoming release:
# https://gitlab.com/gitlab-org/gitlab/issues/196842
end
end
db/schema.rb
View file @ 0a5c6c33
...@@ -2778,7 +2778,6 @@ ActiveRecord::Schema.define(version: 2020_01_27_090233) do ...@@ -2778,7 +2778,6 @@ ActiveRecord::Schema.define(version: 2020_01_27_090233) do
t.index ["commit_id"], name: "index_notes_on_commit_id" t.index ["commit_id"], name: "index_notes_on_commit_id"
t.index ["created_at"], name: "index_notes_on_created_at" t.index ["created_at"], name: "index_notes_on_created_at"
t.index ["discussion_id"], name: "index_notes_on_discussion_id" t.index ["discussion_id"], name: "index_notes_on_discussion_id"
t.index ["id"], name: "epic_mentions_temp_index", where: "((note ~~ '%@%'::text) AND ((noteable_type)::text = 'Epic'::text))"
t.index ["line_code"], name: "index_notes_on_line_code" t.index ["line_code"], name: "index_notes_on_line_code"
t.index ["note"], name: "index_notes_on_note_trigram", opclass: :gin_trgm_ops, using: :gin t.index ["note"], name: "index_notes_on_note_trigram", opclass: :gin_trgm_ops, using: :gin
t.index ["noteable_id", "noteable_type"], name: "index_notes_on_noteable_id_and_noteable_type" t.index ["noteable_id", "noteable_type"], name: "index_notes_on_noteable_id_and_noteable_type"
......
ee/spec/lib/gitlab/background_migration/user_mentions/create_resource_user_mention_spec.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
require 'spec_helper'
require './db/post_migrate/20191115115043_migrate_epic_mentions_to_db'
require './db/post_migrate/20191115115522_migrate_epic_notes_mentions_to_db'
describe Gitlab::BackgroundMigration::UserMentions::CreateResourceUserMention do
include MigrationsHelpers
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) }
let(:notes) { table(:notes) }
let(:epic_user_mentions) { table(:epic_user_mentions) }
let(:author) { users.create!(email: 'author@example.com', notification_email: 'author@example.com', name: 'author', username: 'author', projects_limit: 10, state: 'active') }
let(:member) { users.create!(email: 'member@example.com', notification_email: 'member@example.com', name: 'member', username: 'member', projects_limit: 10, state: 'active') }
let(:admin) { users.create!(email: 'administrator@example.com', notification_email: 'administrator@example.com', name: 'administrator', username: 'administrator', admin: 1, projects_limit: 10, state: 'active') }
let(:john_doe) { users.create!(email: 'john_doe@example.com', notification_email: 'john_doe@example.com', name: 'john_doe', username: 'john_doe', projects_limit: 10, state: 'active') }
let(:skipped) { users.create!(email: 'skipped@example.com', notification_email: 'skipped@example.com', name: 'skipped', username: 'skipped', projects_limit: 10, state: 'active') }
let(:mentioned_users) { [author, member, admin, john_doe, skipped] }
let(:user_mentions) { mentioned_users.map { |u| "@#{u.username}" }.join(' ') }
let(:group) { namespaces.create!(name: 'test1', path: 'test1', runners_token: 'my-token1', project_creation_level: 1, visibility_level: 20, type: 'Group') }
let(:inaccessible_group) { namespaces.create!(name: 'test2', path: 'test2', runners_token: 'my-token2', project_creation_level: 1, visibility_level: 0, type: 'Group') }
let(:mentioned_groups) { [group, inaccessible_group] }
let(:group_mentions) { [group, inaccessible_group].map { |gr| "@#{gr.path}" }.join(' ') }
let(:description_mentions) { "description with mentions #{user_mentions} and #{group_mentions}" }
before do
# build personal namespaces and routes for users
mentioned_users.each { |u| u.becomes(User).save! }
# build namespaces and routes for groups
mentioned_groups.each do |gr|
gr.name += '-org'
gr.path += '-org'
gr.becomes(Namespace).save!
end
end
context 'mentions in epic description' do
let(:epic) do
epics.create!(iid: 1, group_id: group.id, author_id: author.id, title: "epic title @#{author.username}",
title_html: "epic title @#{author.username}", description: description_mentions)
end
it 'migrates mentions' do
join = MigrateEpicMentionsToDb::JOIN
conditions = MigrateEpicMentionsToDb::QUERY_CONDITIONS
expect do
subject.perform('Epic', join, conditions, false, epic.id, epic.id)
end.to change { epic_user_mentions.count }.by(1)
epic_user_mention = epic_user_mentions.last
expect(epic_user_mention.mentioned_users_ids.sort).to eq(mentioned_users.pluck(:id).sort)
expect(epic_user_mention.mentioned_groups_ids.sort).to eq([group.id])
expect(epic_user_mention.mentioned_groups_ids.sort).not_to include(inaccessible_group.id)
end
context 'mentions in epic notes' do
let(:epic_note) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions) }
let(:epic_note2) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: 'sample note') }
let(:system_epic_note) { notes.create!(noteable_id: epic.id, noteable_type: 'Epic', author_id: author.id, note: description_mentions, system: true) }
before do
epic_note.becomes(Note).save!
epic_note2.becomes(Note).save!
system_epic_note.becomes(Note).save!
end
it 'migrates mentions from note' do
join = MigrateEpicNotesMentionsToDb::JOIN
conditions = MigrateEpicNotesMentionsToDb::QUERY_CONDITIONS
expect do
subject.perform('Epic', join, conditions, true, Note.minimum(:id), Note.maximum(:id))
end.to change { epic_user_mentions.where(note_id: epic_note.id).count }.by(1)
# check that the epic_user_mention for regular note is created
epic_user_mention = epic_user_mentions.first
expect(epic_user_mention.becomes(EpicUserMention).note.system).to be false
expect(epic_user_mention.mentioned_users_ids.sort).to eq(users.pluck(:id).sort)
expect(epic_user_mention.mentioned_groups_ids.sort).to eq([group.id])
expect(epic_user_mention.mentioned_groups_ids.sort).not_to include(inaccessible_group.id)
# check that the epic_user_mention for system note is created
epic_user_mention = epic_user_mentions.second
expect(epic_user_mention.becomes(EpicUserMention).note.system).to be true
expect(epic_user_mention.mentioned_users_ids.sort).to eq(users.pluck(:id).sort)
expect(epic_user_mention.mentioned_groups_ids.sort).to eq([group.id])
expect(epic_user_mention.mentioned_groups_ids.sort).not_to include(inaccessible_group.id)
end
end
end
end
ee/spec/migrations/migrate_epic_mentions_to_db_spec.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20191115115043_migrate_epic_mentions_to_db')
describe MigrateEpicMentionsToDb, :migration, :sidekiq do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
users.create!(id: 1, name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0)
namespaces.create!(id: 1, name: 'group1', path: 'group1', owner_id: 1)
epics.create!(id: 1, iid: 1, title: "title1", title_html: 'title1', description: 'epic description with @root mention', group_id: 1, author_id: 1)
epics.create!(id: 2, iid: 2, title: "title2", title_html: "title2", description: 'epic description with @group mention', group_id: 1, author_id: 1)
epics.create!(id: 3, iid: 3, title: "title3", title_html: "title3", description: 'epic description with @project mention', group_id: 1, author_id: 1)
end
it 'schedules epic mentions migrations' do
Sidekiq::Testing.fake! do
Timecop.freeze do
migrate!
migration = described_class::MIGRATION
join = described_class::JOIN
conditions = described_class::QUERY_CONDITIONS
expect(migration).to be_scheduled_delayed_migration(2.minutes, 'Epic', join, conditions, false, 1, 1)
expect(migration).to be_scheduled_delayed_migration(4.minutes, 'Epic', join, conditions, false, 2, 2)
expect(migration).to be_scheduled_delayed_migration(6.minutes, 'Epic', join, conditions, false, 3, 3)
expect(BackgroundMigrationWorker.jobs.size).to eq 3
end
end
end
end
ee/spec/migrations/migrate_epic_notes_mentions_to_db_spec.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20191115115522_migrate_epic_notes_mentions_to_db')
describe MigrateEpicNotesMentionsToDb, :migration, :sidekiq do
let(:users) { table(:users) }
let(:namespaces) { table(:namespaces) }
let(:epics) { table(:epics) }
let(:notes) { table(:notes) }
before do
stub_const("#{described_class.name}::BATCH_SIZE", 1)
users.create!(id: 1, name: 'root', email: 'root@example.com', username: 'root', projects_limit: 0)
namespaces.create!(id: 1, name: 'group1', path: 'group1', owner_id: 1)
epics.create!(id: 1, iid: 1, title: "title", title_html: 'title', description: 'epic description', group_id: 1, author_id: 1)
notes.create!(note: 'note1 for @root to check', noteable_id: 1, noteable_type: 'Epic')
notes.create!(note: 'note2 for @root to check', noteable_id: 1, noteable_type: 'Epic', system: true)
notes.create!(note: 'note3 for @root to check', noteable_id: 1, noteable_type: 'Epic')
end
it 'schedules epic mentions migrations' do
Sidekiq::Testing.fake! do
Timecop.freeze do
migrate!
migration = described_class::MIGRATION
join = described_class::JOIN
conditions = described_class::QUERY_CONDITIONS
expect(migration).to be_scheduled_delayed_migration(2.minutes, 'Epic', join, conditions, true, 1, 1)
expect(migration).to be_scheduled_delayed_migration(4.minutes, 'Epic', join, conditions, true, 2, 2)
expect(migration).to be_scheduled_delayed_migration(6.minutes, 'Epic', join, conditions, true, 3, 3)
expect(BackgroundMigrationWorker.jobs.size).to eq 3
end
end
end
end
lib/gitlab/background_migration/user_mentions/create_resource_user_mention.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
module UserMentions
class CreateResourceUserMention
# Resources that have mentions to be migrated:
# issue, merge_request, epic, commit, snippet, design
BULK_INSERT_SIZE = 5000
ISOLATION_MODULE = 'Gitlab::BackgroundMigration::UserMentions::Models'
def perform(resource_model, join, conditions, with_notes, start_id, end_id)
resource_model = "#{ISOLATION_MODULE}::#{resource_model}".constantize if resource_model.is_a?(String)
model = with_notes ? "#{ISOLATION_MODULE}::Note".constantize : resource_model
resource_user_mention_model = resource_model.user_mention_model
records = model.joins(join).where(conditions).where(id: start_id..end_id)
records.in_groups_of(BULK_INSERT_SIZE, false).each do |records|
mentions = []
records.each do |record|
mentions << record.build_mention_values
end
no_quote_columns = [:note_id]
no_quote_columns << resource_user_mention_model.resource_foreign_key
Gitlab::Database.bulk_insert(
resource_user_mention_model.table_name,
mentions,
return_ids: true,
disable_quote: no_quote_columns,
on_conflict: :do_nothing
)
end
end
end
end
end
end
lib/gitlab/background_migration/user_mentions/models/epic.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
module UserMentions
module Models
class Epic < ActiveRecord::Base
include IsolatedMentionable
include CacheMarkdownField
attr_mentionable :title, pipeline: :single_line
attr_mentionable :description
cache_markdown_field :title, pipeline: :single_line
cache_markdown_field :description, issuable_state_filter_enabled: true
self.table_name = 'epics'
belongs_to :author, class_name: "User"
belongs_to :project
belongs_to :group
def self.user_mention_model
Gitlab::BackgroundMigration::UserMentions::Models::EpicUserMention
end
def user_mention_model
self.class.user_mention_model
end
def project
nil
end
def mentionable_params
{ group: group, label_url_method: :group_epics_url }
end
def user_mention_resource_id
id
end
def user_mention_note_id
'NULL'
end
end
end
end
end
end
lib/gitlab/background_migration/user_mentions/models/epic_user_mention.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
module UserMentions
module Models
class EpicUserMention < ActiveRecord::Base
self.table_name = 'epic_user_mentions'
def self.resource_foreign_key
:epic_id
end
end
end
end
end
end
lib/gitlab/background_migration/user_mentions/models/isolated_mentionable.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
module UserMentions
module Models
# == IsolatedMentionable concern
#
# Shortcutted for isolation version of Mentionable to be used in mentions migrations
#
module IsolatedMentionable
extend ::ActiveSupport::Concern
class_methods do
# Indicate which attributes of the Mentionable to search for GFM references.
def attr_mentionable(attr, options = {})
attr = attr.to_s
mentionable_attrs << [attr, options]
end
end
included do
# Accessor for attributes marked mentionable.
cattr_accessor :mentionable_attrs, instance_accessor: false do
[]
end
if self < Participable
participant -> (user, ext) { all_references(user, extractor: ext) }
end
end
def all_references(current_user = nil, extractor: nil)
# Use custom extractor if it's passed in the function parameters.
if extractor
extractors[current_user] = extractor
else
extractor = extractors[current_user] ||= ::Gitlab::ReferenceExtractor.new(project, current_user)
extractor.reset_memoized_values
end
self.class.mentionable_attrs.each do |attr, options|
text = __send__(attr) # rubocop:disable GitlabSecurity/PublicSend
options = options.merge(
cache_key: [self, attr],
author: author,
skip_project_check: skip_project_check?
).merge(mentionable_params)
cached_html = self.try(:updated_cached_html_for, attr.to_sym)
options[:rendered] = cached_html if cached_html
extractor.analyze(text, options)
end
extractor
end
def extractors
@extractors ||= {}
end
def skip_project_check?
false
end
def build_mention_values
refs = all_references(author)
{
"#{self.user_mention_model.resource_foreign_key}": user_mention_resource_id,
note_id: user_mention_note_id,
mentioned_users_ids: array_to_sql(refs.mentioned_users.pluck(:id)),
mentioned_projects_ids: array_to_sql(refs.mentioned_projects.pluck(:id)),
mentioned_groups_ids: array_to_sql(refs.mentioned_groups.pluck(:id))
}
end
def array_to_sql(ids_array)
return unless ids_array.present?
'{' + ids_array.join(", ") + '}'
end
private
def mentionable_params
{}
end
end
end
end
end
end
lib/gitlab/background_migration/user_mentions/models/note.rb deleted 100644 → 0
View file @ 8240d01a
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
module UserMentions
module Models
class Note < ActiveRecord::Base
include IsolatedMentionable
include CacheMarkdownField
self.table_name = 'notes'
self.inheritance_column = :_type_disabled
attr_mentionable :note, pipeline: :note
cache_markdown_field :note, pipeline: :note, issuable_state_filter_enabled: true
belongs_to :author, class_name: "User"
belongs_to :noteable, polymorphic: true
belongs_to :project
def user_mention_model
"#{CreateResourceUserMention::ISOLATION_MODULE}::#{noteable.class}".constantize.user_mention_model
end
def for_personal_snippet?
noteable.class.name == 'PersonalSnippet'
end
def for_project_noteable?
!for_personal_snippet?
end
def skip_project_check?
!for_project_noteable?
end
def for_epic?
noteable.class.name == 'Epic'
end
def user_mention_resource_id
noteable_id || commit_id
end
def user_mention_note_id
id
end
private
def mentionable_params
return super unless for_epic?
super.merge(banzai_context_params)
end
def banzai_context_params
{ group: noteable.group, label_url_method: :group_epics_url }
end
end
end
end
end
end
spec/support/shared_examples/models/mentionable_shared_examples.rb
View file @ 0a5c6c33
...@@ -229,17 +229,16 @@ RSpec.shared_examples 'mentions in description' do |mentionable_type| ...@@ -229,17 +229,16 @@ RSpec.shared_examples 'mentions in description' do |mentionable_type|
context 'when mentionable description contains mentions' do context 'when mentionable description contains mentions' do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:user2) { create(:user) }
let(:group) { create(:group) } let(:group) { create(:group) }
let(:mentionable_desc) { "#{user.to_reference} #{user2.to_reference} #{user.to_reference} some description #{group.to_reference(full: true)} and #{user2.to_reference} @all" } let(:mentionable_desc) { "#{user.to_reference} some description #{group.to_reference(full: true)} and @all" }
let(:mentionable) { create(mentionable_type, description: mentionable_desc) } let(:mentionable) { create(mentionable_type, description: mentionable_desc) }
it 'stores mentions' do it 'stores mentions' do
add_member(user) add_member(user)
expect(mentionable.user_mentions.count).to eq 1 expect(mentionable.user_mentions.count).to eq 1
expect(mentionable.referenced_users).to match_array([user, user2]) expect(mentionable.referenced_users).to match_array([user])
expect(mentionable.referenced_projects(user)).to match_array([mentionable.project].compact) # epic.project is nil, and we want empty [] expect(mentionable.referenced_projects(user)).to match_array([mentionable.project].compact) # epic.project is nil, and we want empty []
expect(mentionable.referenced_groups(user)).to match_array([group]) expect(mentionable.referenced_groups(user)).to match_array([group])
end end
...@@ -250,9 +249,8 @@ end ...@@ -250,9 +249,8 @@ end
RSpec.shared_examples 'mentions in notes' do |mentionable_type| RSpec.shared_examples 'mentions in notes' do |mentionable_type|
context 'when mentionable notes contain mentions' do context 'when mentionable notes contain mentions' do
let(:user) { create(:user) } let(:user) { create(:user) }
let(:user2) { create(:user) }
let(:group) { create(:group) } let(:group) { create(:group) }
let(:note_desc) { "#{user.to_reference} #{user2.to_reference} #{user.to_reference} and #{group.to_reference(full: true)} and #{user2.to_reference} @all" } let(:note_desc) { "#{user.to_reference} and #{group.to_reference(full: true)} and @all" }
let!(:mentionable) { note.noteable } let!(:mentionable) { note.noteable }
before do before do
...@@ -263,7 +261,7 @@ RSpec.shared_examples 'mentions in notes' do |mentionable_type| ...@@ -263,7 +261,7 @@ RSpec.shared_examples 'mentions in notes' do |mentionable_type|
it 'returns all mentionable mentions' do it 'returns all mentionable mentions' do
expect(mentionable.user_mentions.count).to eq 1 expect(mentionable.user_mentions.count).to eq 1
expect(mentionable.referenced_users).to eq [user, user2] expect(mentionable.referenced_users).to eq [user]
expect(mentionable.referenced_projects(user)).to eq [mentionable.project].compact # epic.project is nil, and we want empty [] expect(mentionable.referenced_projects(user)).to eq [mentionable.project].compact # epic.project is nil, and we want empty []
expect(mentionable.referenced_groups(user)).to eq [group] expect(mentionable.referenced_groups(user)).to eq [group]
end end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7