Commit bef2a919 authored by Valery Sizov's avatar Valery Sizov

Move EE background migrations to CE

All the database related things have to be common
for EE and CE,
including the background migrations.
With this commit we move EE-only backgound migrations
to CE by creating empty ones on CE side and
extending them on EE side.
parent 567b86b2
...@@ -155,11 +155,10 @@ Rails/ApplicationRecord: ...@@ -155,11 +155,10 @@ Rails/ApplicationRecord:
# as they need to be as decoupled from application code as possible # as they need to be as decoupled from application code as possible
- db/**/*.rb - db/**/*.rb
- lib/gitlab/background_migration/**/*.rb - lib/gitlab/background_migration/**/*.rb
- ee/lib/ee/gitlab/background_migration/**/*.rb
- lib/gitlab/database/**/*.rb - lib/gitlab/database/**/*.rb
- spec/**/*.rb - spec/**/*.rb
- ee/db/**/*.rb - ee/db/**/*.rb
- ee/lib/gitlab/background_migration/**/*.rb
- ee/lib/ee/gitlab/background_migration/**/*.rb
- ee/spec/**/*.rb - ee/spec/**/*.rb
# GitLab ################################################################### # GitLab ###################################################################
...@@ -233,7 +232,8 @@ RSpec/FactoriesInMigrationSpecs: ...@@ -233,7 +232,8 @@ RSpec/FactoriesInMigrationSpecs:
- 'spec/migrations/**/*.rb' - 'spec/migrations/**/*.rb'
- 'ee/spec/migrations/**/*.rb' - 'ee/spec/migrations/**/*.rb'
- 'spec/lib/gitlab/background_migration/**/*.rb' - 'spec/lib/gitlab/background_migration/**/*.rb'
- 'ee/spec/lib/gitlab/background_migration/**/*.rb' - 'spec/lib/ee/gitlab/background_migration/**/*.rb'
- 'ee/spec/lib/ee/gitlab/background_migration/**/*.rb'
Cop/IncludeActionViewContext: Cop/IncludeActionViewContext:
Enabled: true Enabled: true
......
---
title: 'Fix: undefined background migration classes for EE-CE downgrades'
merge_request: 22160
author:
type: fixed
# We are extending the original rubocop file for background migrations
# .rubocop.yml file.
---
inherit_from: ../../../../../lib/gitlab/background_migration/.rubocop.yml
CodeReuse/ActiveRecord:
Enabled: false
Style/Documentation:
Enabled: false
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module BackfillVersionDataFromGitaly
extend ::Gitlab::Utils::Override
class Version < ActiveRecord::Base
self.table_name = 'design_management_versions'
self.inheritance_column = :_type_disabled
# The `sha` of a version record must be deserialized from binary
# in order to convert it to a `sha` String that can be used to fetch
# a corresponding Commit from Git.
def sha
value = super
value.unpack1('H*')
end
scope :backfillable_for_issue, -> (issue_id) do
where(author_id: nil).or(where(created_at: nil))
.where(issue_id: issue_id)
end
end
class Issue < ActiveRecord::Base
self.table_name = 'issues'
self.inheritance_column = :_type_disabled
end
override :perform
def perform(issue_id)
issue = Issue.find_by_id(issue_id)
return unless issue
# We need a full Project instance in order to initialize a
# Repository instance that can perform Gitaly calls.
project = ::Project.find_by_id(issue.project_id)
return if project.nil? || project.pending_delete?
# We need a full Repository instance to perform Gitaly calls.
repository = ::DesignManagement::Repository.new(project)
versions = Version.backfillable_for_issue(issue_id)
commits = commits_for_versions(versions, repository)
ActiveRecord::Base.transaction do
versions.each do |version|
commit = commits[version.sha]
unless commit.nil?
version.update_columns(created_at: commit.created_at, author_id: commit.author&.id)
end
end
end
end
private
# Performs a Gitaly request to fetch the corresponding Commit data
# for the given versions.
#
# Returns Commits as a Hash of { sha => Commit }
def commits_for_versions(versions, repository)
shas = versions.map(&:sha)
commits = repository.commits_by(oids: shas)
# Batch load the commit authors so the `User` records are fetched
# all at once the first time we call `commit.author.id`.
commits.each(&:lazy_author)
commits.each_with_object({}) do |commit, hash|
hash[commit.id] = commit
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module GenerateGitlabSubscriptions
extend ::Gitlab::Utils::Override
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled # Disable STI
scope :with_plan, -> { where.not(plan_id: nil) }
scope :without_subscription, -> do
joins("LEFT JOIN gitlab_subscriptions ON namespaces.id = gitlab_subscriptions.namespace_id")
.where(gitlab_subscriptions: { id: nil })
end
def trial_active?
trial_ends_on.present? && trial_ends_on >= Date.today
end
end
class GitlabSubscription < ActiveRecord::Base
self.table_name = 'gitlab_subscriptions'
end
override :perform
def perform(start_id, stop_id)
now = Time.now
# Some fields like seats or end_date will be properly updated by a script executed
# from the subscription portal after this MR hits production.
rows = Namespace
.with_plan
.without_subscription
.where(id: start_id..stop_id)
.select(:id, :plan_id, :trial_ends_on, :created_at)
.map do |namespace|
{
namespace_id: namespace.id,
hosted_plan_id: namespace.plan_id,
trial: namespace.trial_active?,
start_date: namespace.created_at.to_date,
seats: 0,
created_at: now,
updated_at: now
}
end
Gitlab::Database.bulk_insert(:gitlab_subscriptions, rows)
end
end
end
end
end
# frozen_string_literal: true
# rubocop: disable Gitlab/ModuleWithInstanceVariables
module EE
module Gitlab
module BackgroundMigration
# A Project/MergeRequest level migration, aiming to convert existing data
# (from approvers, approver_groups tables)
# to new rule based schema.
module MigrateApproverToApprovalRules
extend ::Gitlab::Utils::Override
include ::Gitlab::Utils::StrongMemoize
class Approver < ActiveRecord::Base
self.table_name = 'approvers'
belongs_to :user
end
class ApproverGroup < ActiveRecord::Base
self.table_name = 'approver_groups'
belongs_to :group
end
class Group < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled
end
class ApprovalMergeRequestRule < ActiveRecord::Base
self.table_name = 'approval_merge_request_rules'
belongs_to :merge_request
scope :code_owner, -> { where(code_owner: true) }
scope :regular, -> { where(code_owner: false) } # Non code owner rule
has_and_belongs_to_many :users
has_and_belongs_to_many :groups, class_name: 'Group', join_table: :approval_merge_request_rules_groups
has_one :approval_merge_request_rule_source
has_one :approval_project_rule, through: :approval_merge_request_rule_source
def project
merge_request.target_project
end
def self.find_or_create_code_owner_rule(merge_request, pattern)
merge_request.approval_rules.safe_find_or_create_by(
code_owner: true,
name: pattern
)
end
def self.safe_find_or_create_by(*args)
safe_ensure_unique(retries: 1) do
find_or_create_by(*args)
end
end
def self.safe_ensure_unique(retries: 0)
transaction(requires_new: true) do
yield
end
rescue ActiveRecord::RecordNotUnique
if retries > 0
retries -= 1
retry
end
false
end
end
class ApprovalMergeRequestRuleSource < ActiveRecord::Base
self.table_name = 'approval_merge_request_rule_sources'
belongs_to :approval_merge_request_rule
belongs_to :approval_project_rule
end
class ApprovalProjectRule < ActiveRecord::Base
self.table_name = 'approval_project_rules'
belongs_to :project
has_and_belongs_to_many :users
has_and_belongs_to_many :groups, class_name: 'Group', join_table: :approval_project_rules_groups
scope :regular, -> { all }
end
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
belongs_to :target_project, class_name: "Project"
has_many :approval_rules, class_name: 'ApprovalMergeRequestRule'
def approvals_required
approvals_before_merge || target_project.approvals_before_merge
end
def distinct(column)
Arel.sql("distinct #{column}")
end
def approver_ids
@approver_ids ||= Approver.where(target_type: 'MergeRequest', target_id: id).joins(:user).pluck(distinct(:user_id))
end
def approver_group_ids
@approver_group_ids ||= ApproverGroup.where(target_type: 'MergeRequest', target_id: id).joins(:group).pluck(distinct(:group_id))
end
def merged_state_id
3
end
def closed_state_id
2
end
def sync_code_owners_with_approvers
return if state_id == merged_state_id || state_id == closed_state_id
::Gitlab::GitalyClient.allow_n_plus_1_calls do
gl_merge_request = ::MergeRequest.find(id)
owners = ::Gitlab::CodeOwners.entries_for_merge_request(gl_merge_request)
.flat_map(&:users).uniq
if owners.present?
ApplicationRecord.transaction do
rule = approval_rules.code_owner.first
rule ||= ApprovalMergeRequestRule.find_or_create_code_owner_rule(
self,
'Code Owner'
)
rule.users = owners.uniq
end
else
approval_rules.code_owner.delete_all
end
end
end
end
class Project < ActiveRecord::Base
self.table_name = 'projects'
has_many :approval_rules, class_name: 'ApprovalProjectRule'
def approver_ids
@approver_ids ||= Approver.where(target_type: 'Project', target_id: id).joins(:user).pluck(Arel.sql('DISTINCT user_id'))
end
def approver_group_ids
@approver_group_ids ||= ApproverGroup.where(target_type: 'Project', target_id: id).joins(:group).pluck(Arel.sql('DISTINCT group_id'))
end
def approvals_required
approvals_before_merge
end
end
class User < ActiveRecord::Base
self.table_name = 'users'
end
ALLOWED_TARGET_TYPES = %w{MergeRequest Project}.freeze
# @param target_type [String] class of target, either 'MergeRequest' or 'Project'
# @param target_id [Integer] id of target
override :perform
def perform(target_type, target_id, sync_code_owner_rule: true)
@target_type = target_type
@target_id = target_id
@sync_code_owner_rule = sync_code_owner_rule
raise "Incorrect target_type #{target_type}" unless ALLOWED_TARGET_TYPES.include?(@target_type)
ActiveRecord::Base.transaction do
case target
when MergeRequest
handle_merge_request
when Project
handle_project
end
end
end
private
def handle_merge_request
if rule = sync_rule
rule.approval_project_rule = target.target_project.approval_rules.regular.first
end
target.sync_code_owners_with_approvers if @sync_code_owner_rule
end
def handle_project
sync_rule
end
def sync_rule
unless approvers_exists?
target.approval_rules.regular.delete_all
return
end
rule = first_or_initialize
rule.update(user_ids: target.approver_ids, group_ids: target.approver_group_ids)
rule
end
def target
strong_memoize(:target) do
case @target_type
when 'MergeRequest'
MergeRequest.find_by(id: @target_id)
when 'Project'
Project.find_by(id: @target_id)
end
end
end
def first_or_initialize
rule = target.approval_rules.regular.first_or_initialize
unless rule.persisted?
rule.name ||= ApprovalRuleLike::DEFAULT_NAME
rule.approvals_required = [target.approvals_required, ApprovalRuleLike::APPROVALS_REQUIRED_MAX].min
rule.save!
end
rule
end
def approvers_exists?
target.approver_ids.any? || target.approver_group_ids.any?
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module MigrateApproverToApprovalRulesCheckProgress
extend ::Gitlab::Utils::Override
RESCHEDULE_DELAY = 1.day
override :perform
def perform
if remaining?
::BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name)
else
::Feature.enable(:approval_rule)
end
end
private
def remaining?
::Gitlab::BackgroundMigration.exists?('MigrateApproverToApprovalRulesInBatch')
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module MigrateApproverToApprovalRulesInBatch
extend ::Gitlab::Utils::Override
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
end
override :perform
def perform(start_id, end_id)
merge_request_ids = MergeRequest.where('id >= ? AND id <= ?', start_id, end_id).pluck(:id)
merge_request_ids.each do |merge_request_id|
::Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.new.perform('MergeRequest', merge_request_id)
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module MoveEpicIssuesAfterEpics
extend ::Gitlab::Utils::Override
class EpicIssue < ActiveRecord::Base
self.table_name = 'epic_issues'
end
class Epic < ActiveRecord::Base
self.table_name = 'epics'
end
override :perform
def perform(start_id, stop_id)
maximum_epic_position = Epic.maximum(:relative_position)
return unless maximum_epic_position
max_position = ::Gitlab::Database::MAX_INT_VALUE
delta = ((maximum_epic_position - max_position) / 2.0).abs.ceil
EpicIssue.where(epic_id: start_id..stop_id).where('relative_position < ?', max_position - delta)
.update_all("relative_position = relative_position + #{delta}")
end
end
end
end
end
# frozen_string_literal: true
# rubocop: disable Gitlab/ModuleWithInstanceVariables
module EE
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given merge request IDs range. A _single_ INSERT is issued for the given range.
module PopulateAnyApprovalRuleForMergeRequests
extend ::Gitlab::Utils::Override
MAX_VALUE = 2**15 - 1
override :perform
def perform(from_id, to_id)
select_sql =
::MergeRequest
.where(merge_request_approval_rules_not_exists_clause)
.where(id: from_id..to_id)
.where('approvals_before_merge <> 0')
.select("id, LEAST(#{MAX_VALUE}, approvals_before_merge), created_at, updated_at, 4, '#{::ApprovalRuleLike::ALL_MEMBERS}'")
.to_sql
execute("INSERT INTO approval_merge_request_rules (merge_request_id, approvals_required, created_at, updated_at, rule_type, name) #{select_sql}")
end
private
def merge_request_approval_rules_not_exists_clause
<<~SQL
NOT EXISTS (SELECT 1 FROM approval_merge_request_rules
WHERE approval_merge_request_rules.merge_request_id = merge_requests.id)
SQL
end
def execute(sql)
@connection ||= ActiveRecord::Base.connection
@connection.execute(sql)
end
end
end
end
end
# frozen_string_literal: true
# rubocop: disable Gitlab/ModuleWithInstanceVariables
module EE
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given project IDs range. A _single_ INSERT is issued for the given range.
module PopulateAnyApprovalRuleForProjects
extend ::Gitlab::Utils::Override
MAX_VALUE = 2**15 - 1
override :perform
def perform(from_id, to_id)
select_sql =
::Project
.where(project_approval_rules_not_exists_clause)
.where(id: from_id..to_id)
.where('approvals_before_merge <> 0')
.select(select_clause)
.to_sql
execute("INSERT INTO approval_project_rules (project_id, approvals_required, created_at, updated_at, rule_type, name) #{select_sql}")
end
private
def select_clause
<<~SQL
id, LEAST(#{MAX_VALUE}, approvals_before_merge),
created_at, updated_at, #{::ApprovalProjectRule.rule_types[:any_approver]}, \'#{ApprovalRuleLike::ALL_MEMBERS}\'
SQL
end
def project_approval_rules_not_exists_clause
<<~SQL
NOT EXISTS (SELECT 1 FROM approval_project_rules
WHERE approval_project_rules.project_id = projects.id)
SQL
end
def execute(sql)
@connection ||= ::ActiveRecord::Base.connection
@connection.execute(sql)
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module PruneOrphanedGeoEvents
extend ::Gitlab::Utils::Override
BATCH_SIZE = 50_000
RESCHEDULE_DELAY = 5.minutes
EVENT_TABLES = %w[geo_repository_created_events
geo_repository_updated_events
geo_repository_deleted_events
geo_repository_renamed_events
geo_repositories_changed_events
geo_hashed_storage_migrated_events
geo_hashed_storage_attachments_events
geo_lfs_object_deleted_events
geo_job_artifact_deleted_events
geo_upload_deleted_events].freeze
module PrunableEvent
extend ActiveSupport::Concern
include EachBatch
included do
scope :orphans, -> do
where(
<<-SQL.squish)
NOT EXISTS (
SELECT 1
FROM geo_event_log
WHERE geo_event_log.#{geo_event_foreign_key} = #{table_name}.id
)
SQL
end
end
class_methods do
def geo_event_foreign_key
table_name.singularize.sub(/^geo_/, '') + '_id'
end
def delete_batch_of_orphans!
deleted = where(id: orphans.limit(BATCH_SIZE)).delete_all
vacuum! if deleted.positive?
deleted
end
def vacuum!
connection.execute("VACUUM #{table_name}")
rescue ActiveRecord::StatementInvalid => e
# ignore timeout, auto-vacuum will take care of it
raise unless e.message =~ /statement timeout/i
end
end
end
override :perform
def perform(table_name = EVENT_TABLES.first)
return if ::Gitlab::Database.read_only?
deleted_rows = prune_orphaned_rows(table_name)
table_name = next_table(table_name) if deleted_rows.zero?
::BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name.demodulize, table_name) if table_name
end
def prune_orphaned_rows(table)
event_model(table).delete_batch_of_orphans!
end
def event_model(table)
Class.new(ActiveRecord::Base) do
include PrunableEvent
self.table_name = table
end
end
def next_table(table_name)
return if EVENT_TABLES.last == table_name
index = EVENT_TABLES.index(table_name)
return unless index
EVENT_TABLES[index + 1]
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module UpdateAuthorizedKeysFileSince
extend ::Gitlab::Utils::Override
include ::Gitlab::ShellAdapter
class Key < ActiveRecord::Base
self.table_name = 'keys'
def shell_id
"key-#{id}"
end
end
delegate :remove_keys_not_found_in_db, to: :gitlab_shell
override :perform
def perform(cutoff_datetime)
add_keys_since(cutoff_datetime)
remove_keys_not_found_in_db
end
def add_keys_since(cutoff_datetime)
start_key = Key.select(:id).where("created_at >= ?", cutoff_datetime).order('id ASC').take
if start_key
batch_add_keys_in_db_starting_from(start_key.id)
end
end
# Not added to Gitlab::Shell because I don't expect this to be used again
def batch_add_keys_in_db_starting_from(start_id)
Rails.logger.info("Adding all keys starting from ID: #{start_id}") # rubocop:disable Gitlab/RailsLogger
Key.find_in_batches(start: start_id, batch_size: 1000) do |keys|
gitlab_shell.batch_add_keys(keys)
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module UpdateVulnerabilityConfidence
extend ::Gitlab::Utils::Override
class Occurrence < ActiveRecord::Base
include ::EachBatch
self.table_name = 'vulnerability_occurrences'
REPORT_TYPES = {
container_scanning: 2
}.freeze
CONFIDENCE_LEVELS = {
unknown: 2,
medium: 5
}.freeze
enum confidences: CONFIDENCE_LEVELS
enum report_type: REPORT_TYPES
def self.container_scanning_reports_with_medium_confidence
where(report_type: self.report_types[:container_scanning], confidence: self.confidences[:medium])
end
end
override :perform
def perform(start_id, stop_id)
Occurrence.container_scanning_reports_with_medium_confidence
.where(id: start_id..stop_id)
.update_all(confidence: Occurrence.confidences[:unknown])
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class BackfillVersionDataFromGitaly
class Version < ActiveRecord::Base
self.table_name = 'design_management_versions'
self.inheritance_column = :_type_disabled
# The `sha` of a version record must be deserialized from binary
# in order to convert it to a `sha` String that can be used to fetch
# a corresponding Commit from Git.
def sha
value = super
value.unpack1('H*')
end
scope :backfillable_for_issue, -> (issue_id) do
where(author_id: nil).or(where(created_at: nil))
.where(issue_id: issue_id)
end
end
class Issue < ActiveRecord::Base
self.table_name = 'issues'
self.inheritance_column = :_type_disabled
end
def perform(issue_id)
issue = Issue.find_by_id(issue_id)
return unless issue
# We need a full Project instance in order to initialize a
# Repository instance that can perform Gitaly calls.
project = Project.find_by_id(issue.project_id)
return if project.nil? || project.pending_delete?
# We need a full Repository instance to perform Gitaly calls.
repository = ::DesignManagement::Repository.new(project)
versions = Version.backfillable_for_issue(issue_id)
commits = commits_for_versions(versions, repository)
ActiveRecord::Base.transaction do
versions.each do |version|
commit = commits[version.sha]
unless commit.nil?
version.update_columns(created_at: commit.created_at, author_id: commit.author&.id)
end
end
end
end
private
# Performs a Gitaly request to fetch the corresponding Commit data
# for the given versions.
#
# Returns Commits as a Hash of { sha => Commit }
def commits_for_versions(versions, repository)
shas = versions.map(&:sha)
commits = repository.commits_by(oids: shas)
# Batch load the commit authors so the `User` records are fetched
# all at once the first time we call `commit.author.id`.
commits.each(&:lazy_author)
commits.each_with_object({}) do |commit, hash|
hash[commit.id] = commit
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class GenerateGitlabSubscriptions
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled # Disable STI
scope :with_plan, -> { where.not(plan_id: nil) }
scope :without_subscription, -> do
joins("LEFT JOIN gitlab_subscriptions ON namespaces.id = gitlab_subscriptions.namespace_id")
.where(gitlab_subscriptions: { id: nil })
end
def trial_active?
trial_ends_on.present? && trial_ends_on >= Date.today
end
end
class GitlabSubscription < ActiveRecord::Base
self.table_name = 'gitlab_subscriptions'
end
def perform(start_id, stop_id)
now = Time.now
# Some fields like seats or end_date will be properly updated by a script executed
# from the subscription portal after this MR hits production.
rows = Namespace
.with_plan
.without_subscription
.where(id: start_id..stop_id)
.select(:id, :plan_id, :trial_ends_on, :created_at)
.map do |namespace|
{
namespace_id: namespace.id,
hosted_plan_id: namespace.plan_id,
trial: namespace.trial_active?,
start_date: namespace.created_at.to_date,
seats: 0,
created_at: now,
updated_at: now
}
end
Gitlab::Database.bulk_insert(:gitlab_subscriptions, rows)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# A Project/MergeRequest level migration, aiming to convert existing data
# (from approvers, approver_groups tables)
# to new rule based schema.
class MigrateApproverToApprovalRules
include Gitlab::Utils::StrongMemoize
class Approver < ActiveRecord::Base
self.table_name = 'approvers'
belongs_to :user
end
class ApproverGroup < ActiveRecord::Base
self.table_name = 'approver_groups'
belongs_to :group
end
class ApprovalMergeRequestRule < ActiveRecord::Base
self.table_name = 'approval_merge_request_rules'
belongs_to :merge_request
scope :code_owner, -> { where(code_owner: true) }
scope :regular, -> { where(code_owner: false) } # Non code owner rule
has_and_belongs_to_many :users
has_and_belongs_to_many :groups, class_name: 'Group', join_table: :approval_merge_request_rules_groups
has_one :approval_merge_request_rule_source
has_one :approval_project_rule, through: :approval_merge_request_rule_source
def project
merge_request.target_project
end
def self.find_or_create_code_owner_rule(merge_request, pattern)
merge_request.approval_rules.safe_find_or_create_by(
code_owner: true,
name: pattern
)
end
def self.safe_find_or_create_by(*args)
safe_ensure_unique(retries: 1) do
find_or_create_by(*args)
end
end
def self.safe_ensure_unique(retries: 0)
transaction(requires_new: true) do
yield
end
rescue ActiveRecord::RecordNotUnique
if retries > 0
retries -= 1
retry
end
false
end
end
class ApprovalMergeRequestRuleSource < ActiveRecord::Base
self.table_name = 'approval_merge_request_rule_sources'
belongs_to :approval_merge_request_rule
belongs_to :approval_project_rule
end
class ApprovalProjectRule < ActiveRecord::Base
self.table_name = 'approval_project_rules'
belongs_to :project
has_and_belongs_to_many :users
has_and_belongs_to_many :groups, class_name: 'Group', join_table: :approval_project_rules_groups
scope :regular, -> { all }
end
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
belongs_to :target_project, class_name: "Project"
has_many :approval_rules, class_name: 'ApprovalMergeRequestRule'
def approvals_required
approvals_before_merge || target_project.approvals_before_merge
end
def distinct(column)
Arel.sql("distinct #{column}")
end
def approver_ids
@approver_ids ||= Approver.where(target_type: 'MergeRequest', target_id: id).joins(:user).pluck(distinct(:user_id))
end
def approver_group_ids
@approver_group_ids ||= ApproverGroup.where(target_type: 'MergeRequest', target_id: id).joins(:group).pluck(distinct(:group_id))
end
def merged_state_id
3
end
def closed_state_id
2
end
def sync_code_owners_with_approvers
return if state_id == merged_state_id || state_id == closed_state_id
Gitlab::GitalyClient.allow_n_plus_1_calls do
gl_merge_request = ::MergeRequest.find(id)
owners = Gitlab::CodeOwners.entries_for_merge_request(gl_merge_request)
.flat_map(&:users).uniq
if owners.present?
ApplicationRecord.transaction do
rule = approval_rules.code_owner.first
rule ||= ApprovalMergeRequestRule.find_or_create_code_owner_rule(
self,
'Code Owner'
)
rule.users = owners.uniq
end
else
approval_rules.code_owner.delete_all
end
end
end
end
class Project < ActiveRecord::Base
self.table_name = 'projects'
has_many :approval_rules, class_name: 'ApprovalProjectRule'
def approver_ids
@approver_ids ||= Approver.where(target_type: 'Project', target_id: id).joins(:user).pluck(Arel.sql('DISTINCT user_id'))
end
def approver_group_ids
@approver_group_ids ||= ApproverGroup.where(target_type: 'Project', target_id: id).joins(:group).pluck(Arel.sql('DISTINCT group_id'))
end
def approvals_required
approvals_before_merge
end
end
class User < ActiveRecord::Base
self.table_name = 'users'
end
ALLOWED_TARGET_TYPES = %w{MergeRequest Project}.freeze
# @param target_type [String] class of target, either 'MergeRequest' or 'Project'
# @param target_id [Integer] id of target
def perform(target_type, target_id, sync_code_owner_rule: true)
@target_type = target_type
@target_id = target_id
@sync_code_owner_rule = sync_code_owner_rule
raise "Incorrect target_type #{target_type}" unless ALLOWED_TARGET_TYPES.include?(@target_type)
ActiveRecord::Base.transaction do
case target
when MergeRequest
handle_merge_request
when Project
handle_project
end
end
end
private
def handle_merge_request
if rule = sync_rule
rule.approval_project_rule = target.target_project.approval_rules.regular.first
end
target.sync_code_owners_with_approvers if @sync_code_owner_rule
end
def handle_project
sync_rule
end
def sync_rule
unless approvers_exists?
target.approval_rules.regular.delete_all
return
end
rule = first_or_initialize
rule.update(user_ids: target.approver_ids, group_ids: target.approver_group_ids)
rule
end
def target
strong_memoize(:target) do
case @target_type
when 'MergeRequest'
MergeRequest.find_by(id: @target_id)
when 'Project'
Project.find_by(id: @target_id)
end
end
end
def first_or_initialize
rule = target.approval_rules.regular.first_or_initialize
unless rule.persisted?
rule.name ||= ApprovalRuleLike::DEFAULT_NAME
rule.approvals_required = [target.approvals_required, ApprovalRuleLike::APPROVALS_REQUIRED_MAX].min
rule.save!
end
rule
end
def approvers_exists?
target.approver_ids.any? || target.approver_group_ids.any?
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class MoveEpicIssuesAfterEpics
class EpicIssue < ActiveRecord::Base
self.table_name = 'epic_issues'
end
class Epic < ActiveRecord::Base
self.table_name = 'epics'
end
def perform(start_id, stop_id)
maximum_epic_position = Epic.maximum(:relative_position)
return unless maximum_epic_position
max_position = Gitlab::Database::MAX_INT_VALUE
delta = ((maximum_epic_position - max_position) / 2.0).abs.ceil
EpicIssue.where(epic_id: start_id..stop_id).where('relative_position < ?', max_position - delta)
.update_all("relative_position = relative_position + #{delta}")
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given merge request IDs range. A _single_ INSERT is issued for the given range.
class PopulateAnyApprovalRuleForMergeRequests
MAX_VALUE = 2**15 - 1
def perform(from_id, to_id)
select_sql =
MergeRequest
.where(merge_request_approval_rules_not_exists_clause)
.where(id: from_id..to_id)
.where('approvals_before_merge <> 0')
.select("id, LEAST(#{MAX_VALUE}, approvals_before_merge), created_at, updated_at, 4, '#{ApprovalRuleLike::ALL_MEMBERS}'")
.to_sql
execute("INSERT INTO approval_merge_request_rules (merge_request_id, approvals_required, created_at, updated_at, rule_type, name) #{select_sql}")
end
private
def merge_request_approval_rules_not_exists_clause
<<~SQL
NOT EXISTS (SELECT 1 FROM approval_merge_request_rules
WHERE approval_merge_request_rules.merge_request_id = merge_requests.id)
SQL
end
def execute(sql)
@connection ||= ActiveRecord::Base.connection
@connection.execute(sql)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given project IDs range. A _single_ INSERT is issued for the given range.
class PopulateAnyApprovalRuleForProjects
MAX_VALUE = 2**15 - 1
def perform(from_id, to_id)
select_sql =
Project
.where(project_approval_rules_not_exists_clause)
.where(id: from_id..to_id)
.where('approvals_before_merge <> 0')
.select("id, LEAST(#{MAX_VALUE}, approvals_before_merge), created_at, updated_at, #{ApprovalProjectRule.rule_types[:any_approver]}, '#{ApprovalRuleLike::ALL_MEMBERS}'")
.to_sql
execute("INSERT INTO approval_project_rules (project_id, approvals_required, created_at, updated_at, rule_type, name) #{select_sql}")
end
private
def project_approval_rules_not_exists_clause
<<~SQL
NOT EXISTS (SELECT 1 FROM approval_project_rules
WHERE approval_project_rules.project_id = projects.id)
SQL
end
def execute(sql)
@connection ||= ActiveRecord::Base.connection
@connection.execute(sql)
end
end
end
end
# frozen_string_literal: true
# This job is added to fix https://gitlab.com/gitlab-org/gitlab/issues/30229
# It's not used anywhere else.
# Can be removed in GitLab 13.*
module Gitlab
module BackgroundMigration
class PruneOrphanedGeoEvents
BATCH_SIZE = 50_000
RESCHEDULE_DELAY = 5.minutes
EVENT_TABLES = %w[geo_repository_created_events
geo_repository_updated_events
geo_repository_deleted_events
geo_repository_renamed_events
geo_repositories_changed_events
geo_hashed_storage_migrated_events
geo_hashed_storage_attachments_events
geo_lfs_object_deleted_events
geo_job_artifact_deleted_events
geo_upload_deleted_events].freeze
module PrunableEvent
extend ActiveSupport::Concern
include EachBatch
included do
scope :orphans, -> do
where(<<-SQL.squish)
NOT EXISTS (
SELECT 1
FROM geo_event_log
WHERE geo_event_log.#{geo_event_foreign_key} = #{table_name}.id
)
SQL
end
end
class_methods do
def geo_event_foreign_key
table_name.singularize.sub(/^geo_/, '') + '_id'
end
def delete_batch_of_orphans!
deleted = where(id: orphans.limit(BATCH_SIZE)).delete_all
vacuum! if deleted.positive?
deleted
end
def vacuum!
connection.execute("VACUUM #{table_name}")
rescue ActiveRecord::StatementInvalid => e
# ignore timeout, auto-vacuum will take care of it
raise unless e.message =~ /statement timeout/i
end
end
end
def perform(table_name = EVENT_TABLES.first)
return if Gitlab::Database.read_only?
deleted_rows = prune_orphaned_rows(table_name)
table_name = next_table(table_name) if deleted_rows.zero?
BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name.demodulize, table_name) if table_name
end
def prune_orphaned_rows(table)
event_model(table).delete_batch_of_orphans!
end
def event_model(table)
Class.new(ActiveRecord::Base) do
include PrunableEvent
self.table_name = table
end
end
def next_table(table_name)
return if EVENT_TABLES.last == table_name
index = EVENT_TABLES.index(table_name)
return unless index
EVENT_TABLES[index + 1]
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class UpdateAuthorizedKeysFileSince
include Gitlab::ShellAdapter
class Key < ActiveRecord::Base
self.table_name = 'keys'
def shell_id
"key-#{id}"
end
end
delegate :remove_keys_not_found_in_db, to: :gitlab_shell
def perform(cutoff_datetime)
add_keys_since(cutoff_datetime)
remove_keys_not_found_in_db
end
def add_keys_since(cutoff_datetime)
start_key = Key.select(:id).where("created_at >= ?", cutoff_datetime).order('id ASC').take
if start_key
batch_add_keys_in_db_starting_from(start_key.id)
end
end
# Not added to Gitlab::Shell because I don't expect this to be used again
def batch_add_keys_in_db_starting_from(start_id)
Rails.logger.info("Adding all keys starting from ID: #{start_id}") # rubocop:disable Gitlab/RailsLogger
::Key.find_in_batches(start: start_id, batch_size: 1000) do |keys|
gitlab_shell.batch_add_keys(keys)
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class UpdateVulnerabilityConfidence
class Occurrence < ActiveRecord::Base
include ::EachBatch
self.table_name = 'vulnerability_occurrences'
REPORT_TYPES = {
container_scanning: 2
}.freeze
CONFIDENCE_LEVELS = {
unknown: 2,
medium: 5
}.freeze
enum confidences: CONFIDENCE_LEVELS
enum report_type: REPORT_TYPES
def self.container_scanning_reports_with_medium_confidence
where(report_type: self.report_types[:container_scanning], confidence: self.confidences[:medium])
end
end
def perform(start_id, stop_id)
Occurrence.container_scanning_reports_with_medium_confidence
.where(id: start_id..stop_id)
.update_all(confidence: Occurrence.confidences[:unknown])
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class BackfillVersionDataFromGitaly
def perform(issue_id)
end
end
end
end
Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_if_ee('EE::Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class GenerateGitlabSubscriptions
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_if_ee('EE::Gitlab::BackgroundMigration::GenerateGitlabSubscriptions')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class MigrateApproverToApprovalRules
# @param target_type [String] class of target, either 'MergeRequest' or 'Project'
# @param target_id [Integer] id of target
def perform(target_type, target_id, sync_code_owner_rule: true)
end
end
end
end
Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRules')
...@@ -2,22 +2,12 @@ ...@@ -2,22 +2,12 @@
module Gitlab module Gitlab
module BackgroundMigration module BackgroundMigration
# rubocop: disable Style/Documentation
class MigrateApproverToApprovalRulesCheckProgress class MigrateApproverToApprovalRulesCheckProgress
RESCHEDULE_DELAY = 1.day
def perform def perform
if remaining?
BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name)
else
Feature.enable(:approval_rule)
end
end
private
def remaining?
Gitlab::BackgroundMigration.exists?('MigrateApproverToApprovalRulesInBatch')
end end
end end
end end
end end
Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress')
...@@ -2,17 +2,12 @@ ...@@ -2,17 +2,12 @@
module Gitlab module Gitlab
module BackgroundMigration module BackgroundMigration
# rubocop: disable Style/Documentation
class MigrateApproverToApprovalRulesInBatch class MigrateApproverToApprovalRulesInBatch
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
end
def perform(start_id, end_id) def perform(start_id, end_id)
merge_request_ids = MergeRequest.where('id >= ? AND id <= ?', start_id, end_id).pluck(:id)
merge_request_ids.each do |merge_request_id|
MigrateApproverToApprovalRules.new.perform('MergeRequest', merge_request_id)
end
end end
end end
end end
end end
Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class MoveEpicIssuesAfterEpics
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_if_ee('EE::Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given merge request IDs range. A _single_ INSERT is issued for the given range.
class PopulateAnyApprovalRuleForMergeRequests
def perform(from_id, to_id)
end
end
end
end
Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given project IDs range. A _single_ INSERT is issued for the given range.
class PopulateAnyApprovalRuleForProjects
def perform(from_id, to_id)
end
end
end
end
Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects')
# frozen_string_literal: true
#
# rubocop:disable Style/Documentation
# This job is added to fix https://gitlab.com/gitlab-org/gitlab/issues/30229
# It's not used anywhere else.
# Can be removed in GitLab 13.*
module Gitlab
module BackgroundMigration
class PruneOrphanedGeoEvents
def perform(table_name)
end
end
end
end
Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::PruneOrphanedGeoEvents')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class UpdateAuthorizedKeysFileSince
def perform(cutoff_datetime)
end
end
end
end
Gitlab::BackgroundMigration::UpdateAuthorizedKeysFileSince.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateAuthorizedKeysFileSince')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class UpdateVulnerabilityConfidence
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment