Commit 77a84ebe authored by Dmytro Zaporozhets's avatar Dmytro Zaporozhets

Merge branch...

Merge branch '30229-background-migration-pruneorphanedgeoevents-did-you-mean-pruneoldeventsworker' into 'master'

Fix: undefined background migration classes for EE-CE downgrade

Closes #30229

See merge request gitlab-org/gitlab!22160
parents 11311bd5 bef2a919
...@@ -155,11 +155,10 @@ Rails/ApplicationRecord: ...@@ -155,11 +155,10 @@ Rails/ApplicationRecord:
# as they need to be as decoupled from application code as possible # as they need to be as decoupled from application code as possible
- db/**/*.rb - db/**/*.rb
- lib/gitlab/background_migration/**/*.rb - lib/gitlab/background_migration/**/*.rb
- ee/lib/ee/gitlab/background_migration/**/*.rb
- lib/gitlab/database/**/*.rb - lib/gitlab/database/**/*.rb
- spec/**/*.rb - spec/**/*.rb
- ee/db/**/*.rb - ee/db/**/*.rb
- ee/lib/gitlab/background_migration/**/*.rb
- ee/lib/ee/gitlab/background_migration/**/*.rb
- ee/spec/**/*.rb - ee/spec/**/*.rb
# GitLab ################################################################### # GitLab ###################################################################
...@@ -233,7 +232,8 @@ RSpec/FactoriesInMigrationSpecs: ...@@ -233,7 +232,8 @@ RSpec/FactoriesInMigrationSpecs:
- 'spec/migrations/**/*.rb' - 'spec/migrations/**/*.rb'
- 'ee/spec/migrations/**/*.rb' - 'ee/spec/migrations/**/*.rb'
- 'spec/lib/gitlab/background_migration/**/*.rb' - 'spec/lib/gitlab/background_migration/**/*.rb'
- 'ee/spec/lib/gitlab/background_migration/**/*.rb' - 'spec/lib/ee/gitlab/background_migration/**/*.rb'
- 'ee/spec/lib/ee/gitlab/background_migration/**/*.rb'
Cop/IncludeActionViewContext: Cop/IncludeActionViewContext:
Enabled: true Enabled: true
......
---
title: 'Fix: undefined background migration classes for EE-CE downgrades'
merge_request: 22160
author:
type: fixed
# We are extending the original rubocop file for background migrations
# .rubocop.yml file.
---
inherit_from: ../../../../../lib/gitlab/background_migration/.rubocop.yml
CodeReuse/ActiveRecord:
Enabled: false
Style/Documentation:
Enabled: false
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module BackfillVersionDataFromGitaly
extend ::Gitlab::Utils::Override
class Version < ActiveRecord::Base
self.table_name = 'design_management_versions'
self.inheritance_column = :_type_disabled
# The `sha` of a version record must be deserialized from binary
# in order to convert it to a `sha` String that can be used to fetch
# a corresponding Commit from Git.
def sha
value = super
value.unpack1('H*')
end
scope :backfillable_for_issue, -> (issue_id) do
where(author_id: nil).or(where(created_at: nil))
.where(issue_id: issue_id)
end
end
class Issue < ActiveRecord::Base
self.table_name = 'issues'
self.inheritance_column = :_type_disabled
end
override :perform
def perform(issue_id)
issue = Issue.find_by_id(issue_id)
return unless issue
# We need a full Project instance in order to initialize a
# Repository instance that can perform Gitaly calls.
project = ::Project.find_by_id(issue.project_id)
return if project.nil? || project.pending_delete?
# We need a full Repository instance to perform Gitaly calls.
repository = ::DesignManagement::Repository.new(project)
versions = Version.backfillable_for_issue(issue_id)
commits = commits_for_versions(versions, repository)
ActiveRecord::Base.transaction do
versions.each do |version|
commit = commits[version.sha]
unless commit.nil?
version.update_columns(created_at: commit.created_at, author_id: commit.author&.id)
end
end
end
end
private
# Performs a Gitaly request to fetch the corresponding Commit data
# for the given versions.
#
# Returns Commits as a Hash of { sha => Commit }
def commits_for_versions(versions, repository)
shas = versions.map(&:sha)
commits = repository.commits_by(oids: shas)
# Batch load the commit authors so the `User` records are fetched
# all at once the first time we call `commit.author.id`.
commits.each(&:lazy_author)
commits.each_with_object({}) do |commit, hash|
hash[commit.id] = commit
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module GenerateGitlabSubscriptions
extend ::Gitlab::Utils::Override
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled # Disable STI
scope :with_plan, -> { where.not(plan_id: nil) }
scope :without_subscription, -> do
joins("LEFT JOIN gitlab_subscriptions ON namespaces.id = gitlab_subscriptions.namespace_id")
.where(gitlab_subscriptions: { id: nil })
end
def trial_active?
trial_ends_on.present? && trial_ends_on >= Date.today
end
end
class GitlabSubscription < ActiveRecord::Base
self.table_name = 'gitlab_subscriptions'
end
override :perform
def perform(start_id, stop_id)
now = Time.now
# Some fields like seats or end_date will be properly updated by a script executed
# from the subscription portal after this MR hits production.
rows = Namespace
.with_plan
.without_subscription
.where(id: start_id..stop_id)
.select(:id, :plan_id, :trial_ends_on, :created_at)
.map do |namespace|
{
namespace_id: namespace.id,
hosted_plan_id: namespace.plan_id,
trial: namespace.trial_active?,
start_date: namespace.created_at.to_date,
seats: 0,
created_at: now,
updated_at: now
}
end
Gitlab::Database.bulk_insert(:gitlab_subscriptions, rows)
end
end
end
end
end
# frozen_string_literal: true
# rubocop: disable Gitlab/ModuleWithInstanceVariables
module EE
module Gitlab
module BackgroundMigration
# A Project/MergeRequest level migration, aiming to convert existing data
# (from approvers, approver_groups tables)
# to new rule based schema.
module MigrateApproverToApprovalRules
extend ::Gitlab::Utils::Override
include ::Gitlab::Utils::StrongMemoize
class Approver < ActiveRecord::Base
self.table_name = 'approvers'
belongs_to :user
end
class ApproverGroup < ActiveRecord::Base
self.table_name = 'approver_groups'
belongs_to :group
end
class Group < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled
end
class ApprovalMergeRequestRule < ActiveRecord::Base
self.table_name = 'approval_merge_request_rules'
belongs_to :merge_request
scope :code_owner, -> { where(code_owner: true) }
scope :regular, -> { where(code_owner: false) } # Non code owner rule
has_and_belongs_to_many :users
has_and_belongs_to_many :groups, class_name: 'Group', join_table: :approval_merge_request_rules_groups
has_one :approval_merge_request_rule_source
has_one :approval_project_rule, through: :approval_merge_request_rule_source
def project
merge_request.target_project
end
def self.find_or_create_code_owner_rule(merge_request, pattern)
merge_request.approval_rules.safe_find_or_create_by(
code_owner: true,
name: pattern
)
end
def self.safe_find_or_create_by(*args)
safe_ensure_unique(retries: 1) do
find_or_create_by(*args)
end
end
def self.safe_ensure_unique(retries: 0)
transaction(requires_new: true) do
yield
end
rescue ActiveRecord::RecordNotUnique
if retries > 0
retries -= 1
retry
end
false
end
end
class ApprovalMergeRequestRuleSource < ActiveRecord::Base
self.table_name = 'approval_merge_request_rule_sources'
belongs_to :approval_merge_request_rule
belongs_to :approval_project_rule
end
class ApprovalProjectRule < ActiveRecord::Base
self.table_name = 'approval_project_rules'
belongs_to :project
has_and_belongs_to_many :users
has_and_belongs_to_many :groups, class_name: 'Group', join_table: :approval_project_rules_groups
scope :regular, -> { all }
end
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
belongs_to :target_project, class_name: "Project"
has_many :approval_rules, class_name: 'ApprovalMergeRequestRule'
def approvals_required
approvals_before_merge || target_project.approvals_before_merge
end
def distinct(column)
Arel.sql("distinct #{column}")
end
def approver_ids
@approver_ids ||= Approver.where(target_type: 'MergeRequest', target_id: id).joins(:user).pluck(distinct(:user_id))
end
def approver_group_ids
@approver_group_ids ||= ApproverGroup.where(target_type: 'MergeRequest', target_id: id).joins(:group).pluck(distinct(:group_id))
end
def merged_state_id
3
end
def closed_state_id
2
end
def sync_code_owners_with_approvers
return if state_id == merged_state_id || state_id == closed_state_id
::Gitlab::GitalyClient.allow_n_plus_1_calls do
gl_merge_request = ::MergeRequest.find(id)
owners = ::Gitlab::CodeOwners.entries_for_merge_request(gl_merge_request)
.flat_map(&:users).uniq
if owners.present?
ApplicationRecord.transaction do
rule = approval_rules.code_owner.first
rule ||= ApprovalMergeRequestRule.find_or_create_code_owner_rule(
self,
'Code Owner'
)
rule.users = owners.uniq
end
else
approval_rules.code_owner.delete_all
end
end
end
end
class Project < ActiveRecord::Base
self.table_name = 'projects'
has_many :approval_rules, class_name: 'ApprovalProjectRule'
def approver_ids
@approver_ids ||= Approver.where(target_type: 'Project', target_id: id).joins(:user).pluck(Arel.sql('DISTINCT user_id'))
end
def approver_group_ids
@approver_group_ids ||= ApproverGroup.where(target_type: 'Project', target_id: id).joins(:group).pluck(Arel.sql('DISTINCT group_id'))
end
def approvals_required
approvals_before_merge
end
end
class User < ActiveRecord::Base
self.table_name = 'users'
end
ALLOWED_TARGET_TYPES = %w{MergeRequest Project}.freeze
# @param target_type [String] class of target, either 'MergeRequest' or 'Project'
# @param target_id [Integer] id of target
override :perform
def perform(target_type, target_id, sync_code_owner_rule: true)
@target_type = target_type
@target_id = target_id
@sync_code_owner_rule = sync_code_owner_rule
raise "Incorrect target_type #{target_type}" unless ALLOWED_TARGET_TYPES.include?(@target_type)
ActiveRecord::Base.transaction do
case target
when MergeRequest
handle_merge_request
when Project
handle_project
end
end
end
private
def handle_merge_request
if rule = sync_rule
rule.approval_project_rule = target.target_project.approval_rules.regular.first
end
target.sync_code_owners_with_approvers if @sync_code_owner_rule
end
def handle_project
sync_rule
end
def sync_rule
unless approvers_exists?
target.approval_rules.regular.delete_all
return
end
rule = first_or_initialize
rule.update(user_ids: target.approver_ids, group_ids: target.approver_group_ids)
rule
end
def target
strong_memoize(:target) do
case @target_type
when 'MergeRequest'
MergeRequest.find_by(id: @target_id)
when 'Project'
Project.find_by(id: @target_id)
end
end
end
def first_or_initialize
rule = target.approval_rules.regular.first_or_initialize
unless rule.persisted?
rule.name ||= ApprovalRuleLike::DEFAULT_NAME
rule.approvals_required = [target.approvals_required, ApprovalRuleLike::APPROVALS_REQUIRED_MAX].min
rule.save!
end
rule
end
def approvers_exists?
target.approver_ids.any? || target.approver_group_ids.any?
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module MigrateApproverToApprovalRulesCheckProgress
extend ::Gitlab::Utils::Override
RESCHEDULE_DELAY = 1.day
override :perform
def perform
if remaining?
::BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name)
else
::Feature.enable(:approval_rule)
end
end
private
def remaining?
::Gitlab::BackgroundMigration.exists?('MigrateApproverToApprovalRulesInBatch')
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module MigrateApproverToApprovalRulesInBatch
extend ::Gitlab::Utils::Override
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
end
override :perform
def perform(start_id, end_id)
merge_request_ids = MergeRequest.where('id >= ? AND id <= ?', start_id, end_id).pluck(:id)
merge_request_ids.each do |merge_request_id|
::Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.new.perform('MergeRequest', merge_request_id)
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module MoveEpicIssuesAfterEpics
extend ::Gitlab::Utils::Override
class EpicIssue < ActiveRecord::Base
self.table_name = 'epic_issues'
end
class Epic < ActiveRecord::Base
self.table_name = 'epics'
end
override :perform
def perform(start_id, stop_id)
maximum_epic_position = Epic.maximum(:relative_position)
return unless maximum_epic_position
max_position = ::Gitlab::Database::MAX_INT_VALUE
delta = ((maximum_epic_position - max_position) / 2.0).abs.ceil
EpicIssue.where(epic_id: start_id..stop_id).where('relative_position < ?', max_position - delta)
.update_all("relative_position = relative_position + #{delta}")
end
end
end
end
end
# frozen_string_literal: true
# rubocop: disable Gitlab/ModuleWithInstanceVariables
module EE
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given merge request IDs range. A _single_ INSERT is issued for the given range.
module PopulateAnyApprovalRuleForMergeRequests
extend ::Gitlab::Utils::Override
MAX_VALUE = 2**15 - 1
override :perform
def perform(from_id, to_id)
select_sql =
::MergeRequest
.where(merge_request_approval_rules_not_exists_clause)
.where(id: from_id..to_id)
.where('approvals_before_merge <> 0')
.select("id, LEAST(#{MAX_VALUE}, approvals_before_merge), created_at, updated_at, 4, '#{::ApprovalRuleLike::ALL_MEMBERS}'")
.to_sql
execute("INSERT INTO approval_merge_request_rules (merge_request_id, approvals_required, created_at, updated_at, rule_type, name) #{select_sql}")
end
private
def merge_request_approval_rules_not_exists_clause
<<~SQL
NOT EXISTS (SELECT 1 FROM approval_merge_request_rules
WHERE approval_merge_request_rules.merge_request_id = merge_requests.id)
SQL
end
def execute(sql)
@connection ||= ActiveRecord::Base.connection
@connection.execute(sql)
end
end
end
end
end
# frozen_string_literal: true
# rubocop: disable Gitlab/ModuleWithInstanceVariables
module EE
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given project IDs range. A _single_ INSERT is issued for the given range.
module PopulateAnyApprovalRuleForProjects
extend ::Gitlab::Utils::Override
MAX_VALUE = 2**15 - 1
override :perform
def perform(from_id, to_id)
select_sql =
::Project
.where(project_approval_rules_not_exists_clause)
.where(id: from_id..to_id)
.where('approvals_before_merge <> 0')
.select(select_clause)
.to_sql
execute("INSERT INTO approval_project_rules (project_id, approvals_required, created_at, updated_at, rule_type, name) #{select_sql}")
end
private
def select_clause
<<~SQL
id, LEAST(#{MAX_VALUE}, approvals_before_merge),
created_at, updated_at, #{::ApprovalProjectRule.rule_types[:any_approver]}, \'#{ApprovalRuleLike::ALL_MEMBERS}\'
SQL
end
def project_approval_rules_not_exists_clause
<<~SQL
NOT EXISTS (SELECT 1 FROM approval_project_rules
WHERE approval_project_rules.project_id = projects.id)
SQL
end
def execute(sql)
@connection ||= ::ActiveRecord::Base.connection
@connection.execute(sql)
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module PruneOrphanedGeoEvents
extend ::Gitlab::Utils::Override
BATCH_SIZE = 50_000
RESCHEDULE_DELAY = 5.minutes
EVENT_TABLES = %w[geo_repository_created_events
geo_repository_updated_events
geo_repository_deleted_events
geo_repository_renamed_events
geo_repositories_changed_events
geo_hashed_storage_migrated_events
geo_hashed_storage_attachments_events
geo_lfs_object_deleted_events
geo_job_artifact_deleted_events
geo_upload_deleted_events].freeze
module PrunableEvent
extend ActiveSupport::Concern
include EachBatch
included do
scope :orphans, -> do
where(
<<-SQL.squish)
NOT EXISTS (
SELECT 1
FROM geo_event_log
WHERE geo_event_log.#{geo_event_foreign_key} = #{table_name}.id
)
SQL
end
end
class_methods do
def geo_event_foreign_key
table_name.singularize.sub(/^geo_/, '') + '_id'
end
def delete_batch_of_orphans!
deleted = where(id: orphans.limit(BATCH_SIZE)).delete_all
vacuum! if deleted.positive?
deleted
end
def vacuum!
connection.execute("VACUUM #{table_name}")
rescue ActiveRecord::StatementInvalid => e
# ignore timeout, auto-vacuum will take care of it
raise unless e.message =~ /statement timeout/i
end
end
end
override :perform
def perform(table_name = EVENT_TABLES.first)
return if ::Gitlab::Database.read_only?
deleted_rows = prune_orphaned_rows(table_name)
table_name = next_table(table_name) if deleted_rows.zero?
::BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name.demodulize, table_name) if table_name
end
def prune_orphaned_rows(table)
event_model(table).delete_batch_of_orphans!
end
def event_model(table)
Class.new(ActiveRecord::Base) do
include PrunableEvent
self.table_name = table
end
end
def next_table(table_name)
return if EVENT_TABLES.last == table_name
index = EVENT_TABLES.index(table_name)
return unless index
EVENT_TABLES[index + 1]
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module UpdateAuthorizedKeysFileSince
extend ::Gitlab::Utils::Override
include ::Gitlab::ShellAdapter
class Key < ActiveRecord::Base
self.table_name = 'keys'
def shell_id
"key-#{id}"
end
end
delegate :remove_keys_not_found_in_db, to: :gitlab_shell
override :perform
def perform(cutoff_datetime)
add_keys_since(cutoff_datetime)
remove_keys_not_found_in_db
end
def add_keys_since(cutoff_datetime)
start_key = Key.select(:id).where("created_at >= ?", cutoff_datetime).order('id ASC').take
if start_key
batch_add_keys_in_db_starting_from(start_key.id)
end
end
# Not added to Gitlab::Shell because I don't expect this to be used again
def batch_add_keys_in_db_starting_from(start_id)
Rails.logger.info("Adding all keys starting from ID: #{start_id}") # rubocop:disable Gitlab/RailsLogger
Key.find_in_batches(start: start_id, batch_size: 1000) do |keys|
gitlab_shell.batch_add_keys(keys)
end
end
end
end
end
end
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module UpdateVulnerabilityConfidence
extend ::Gitlab::Utils::Override
class Occurrence < ActiveRecord::Base
include ::EachBatch
self.table_name = 'vulnerability_occurrences'
REPORT_TYPES = {
container_scanning: 2
}.freeze
CONFIDENCE_LEVELS = {
unknown: 2,
medium: 5
}.freeze
enum confidences: CONFIDENCE_LEVELS
enum report_type: REPORT_TYPES
def self.container_scanning_reports_with_medium_confidence
where(report_type: self.report_types[:container_scanning], confidence: self.confidences[:medium])
end
end
override :perform
def perform(start_id, stop_id)
Occurrence.container_scanning_reports_with_medium_confidence
.where(id: start_id..stop_id)
.update_all(confidence: Occurrence.confidences[:unknown])
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class BackfillVersionDataFromGitaly
class Version < ActiveRecord::Base
self.table_name = 'design_management_versions'
self.inheritance_column = :_type_disabled
# The `sha` of a version record must be deserialized from binary
# in order to convert it to a `sha` String that can be used to fetch
# a corresponding Commit from Git.
def sha
value = super
value.unpack1('H*')
end
scope :backfillable_for_issue, -> (issue_id) do
where(author_id: nil).or(where(created_at: nil))
.where(issue_id: issue_id)
end
end
class Issue < ActiveRecord::Base
self.table_name = 'issues'
self.inheritance_column = :_type_disabled
end
def perform(issue_id)
issue = Issue.find_by_id(issue_id)
return unless issue
# We need a full Project instance in order to initialize a
# Repository instance that can perform Gitaly calls.
project = Project.find_by_id(issue.project_id)
return if project.nil? || project.pending_delete?
# We need a full Repository instance to perform Gitaly calls.
repository = ::DesignManagement::Repository.new(project)
versions = Version.backfillable_for_issue(issue_id)
commits = commits_for_versions(versions, repository)
ActiveRecord::Base.transaction do
versions.each do |version|
commit = commits[version.sha]
unless commit.nil?
version.update_columns(created_at: commit.created_at, author_id: commit.author&.id)
end
end
end
end
private
# Performs a Gitaly request to fetch the corresponding Commit data
# for the given versions.
#
# Returns Commits as a Hash of { sha => Commit }
def commits_for_versions(versions, repository)
shas = versions.map(&:sha)
commits = repository.commits_by(oids: shas)
# Batch load the commit authors so the `User` records are fetched
# all at once the first time we call `commit.author.id`.
commits.each(&:lazy_author)
commits.each_with_object({}) do |commit, hash|
hash[commit.id] = commit
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class GenerateGitlabSubscriptions
class Namespace < ActiveRecord::Base
self.table_name = 'namespaces'
self.inheritance_column = :_type_disabled # Disable STI
scope :with_plan, -> { where.not(plan_id: nil) }
scope :without_subscription, -> do
joins("LEFT JOIN gitlab_subscriptions ON namespaces.id = gitlab_subscriptions.namespace_id")
.where(gitlab_subscriptions: { id: nil })
end
def trial_active?
trial_ends_on.present? && trial_ends_on >= Date.today
end
end
class GitlabSubscription < ActiveRecord::Base
self.table_name = 'gitlab_subscriptions'
end
def perform(start_id, stop_id)
now = Time.now
# Some fields like seats or end_date will be properly updated by a script executed
# from the subscription portal after this MR hits production.
rows = Namespace
.with_plan
.without_subscription
.where(id: start_id..stop_id)
.select(:id, :plan_id, :trial_ends_on, :created_at)
.map do |namespace|
{
namespace_id: namespace.id,
hosted_plan_id: namespace.plan_id,
trial: namespace.trial_active?,
start_date: namespace.created_at.to_date,
seats: 0,
created_at: now,
updated_at: now
}
end
Gitlab::Database.bulk_insert(:gitlab_subscriptions, rows)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# A Project/MergeRequest level migration, aiming to convert existing data
# (from approvers, approver_groups tables)
# to new rule based schema.
class MigrateApproverToApprovalRules
include Gitlab::Utils::StrongMemoize
class Approver < ActiveRecord::Base
self.table_name = 'approvers'
belongs_to :user
end
class ApproverGroup < ActiveRecord::Base
self.table_name = 'approver_groups'
belongs_to :group
end
class ApprovalMergeRequestRule < ActiveRecord::Base
self.table_name = 'approval_merge_request_rules'
belongs_to :merge_request
scope :code_owner, -> { where(code_owner: true) }
scope :regular, -> { where(code_owner: false) } # Non code owner rule
has_and_belongs_to_many :users
has_and_belongs_to_many :groups, class_name: 'Group', join_table: :approval_merge_request_rules_groups
has_one :approval_merge_request_rule_source
has_one :approval_project_rule, through: :approval_merge_request_rule_source
def project
merge_request.target_project
end
def self.find_or_create_code_owner_rule(merge_request, pattern)
merge_request.approval_rules.safe_find_or_create_by(
code_owner: true,
name: pattern
)
end
def self.safe_find_or_create_by(*args)
safe_ensure_unique(retries: 1) do
find_or_create_by(*args)
end
end
def self.safe_ensure_unique(retries: 0)
transaction(requires_new: true) do
yield
end
rescue ActiveRecord::RecordNotUnique
if retries > 0
retries -= 1
retry
end
false
end
end
class ApprovalMergeRequestRuleSource < ActiveRecord::Base
self.table_name = 'approval_merge_request_rule_sources'
belongs_to :approval_merge_request_rule
belongs_to :approval_project_rule
end
class ApprovalProjectRule < ActiveRecord::Base
self.table_name = 'approval_project_rules'
belongs_to :project
has_and_belongs_to_many :users
has_and_belongs_to_many :groups, class_name: 'Group', join_table: :approval_project_rules_groups
scope :regular, -> { all }
end
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
belongs_to :target_project, class_name: "Project"
has_many :approval_rules, class_name: 'ApprovalMergeRequestRule'
def approvals_required
approvals_before_merge || target_project.approvals_before_merge
end
def distinct(column)
Arel.sql("distinct #{column}")
end
def approver_ids
@approver_ids ||= Approver.where(target_type: 'MergeRequest', target_id: id).joins(:user).pluck(distinct(:user_id))
end
def approver_group_ids
@approver_group_ids ||= ApproverGroup.where(target_type: 'MergeRequest', target_id: id).joins(:group).pluck(distinct(:group_id))
end
def merged_state_id
3
end
def closed_state_id
2
end
def sync_code_owners_with_approvers
return if state_id == merged_state_id || state_id == closed_state_id
Gitlab::GitalyClient.allow_n_plus_1_calls do
gl_merge_request = ::MergeRequest.find(id)
owners = Gitlab::CodeOwners.entries_for_merge_request(gl_merge_request)
.flat_map(&:users).uniq
if owners.present?
ApplicationRecord.transaction do
rule = approval_rules.code_owner.first
rule ||= ApprovalMergeRequestRule.find_or_create_code_owner_rule(
self,
'Code Owner'
)
rule.users = owners.uniq
end
else
approval_rules.code_owner.delete_all
end
end
end
end
class Project < ActiveRecord::Base
self.table_name = 'projects'
has_many :approval_rules, class_name: 'ApprovalProjectRule'
def approver_ids
@approver_ids ||= Approver.where(target_type: 'Project', target_id: id).joins(:user).pluck(Arel.sql('DISTINCT user_id'))
end
def approver_group_ids
@approver_group_ids ||= ApproverGroup.where(target_type: 'Project', target_id: id).joins(:group).pluck(Arel.sql('DISTINCT group_id'))
end
def approvals_required
approvals_before_merge
end
end
class User < ActiveRecord::Base
self.table_name = 'users'
end
ALLOWED_TARGET_TYPES = %w{MergeRequest Project}.freeze
# @param target_type [String] class of target, either 'MergeRequest' or 'Project'
# @param target_id [Integer] id of target
def perform(target_type, target_id, sync_code_owner_rule: true)
@target_type = target_type
@target_id = target_id
@sync_code_owner_rule = sync_code_owner_rule
raise "Incorrect target_type #{target_type}" unless ALLOWED_TARGET_TYPES.include?(@target_type)
ActiveRecord::Base.transaction do
case target
when MergeRequest
handle_merge_request
when Project
handle_project
end
end
end
private
def handle_merge_request
if rule = sync_rule
rule.approval_project_rule = target.target_project.approval_rules.regular.first
end
target.sync_code_owners_with_approvers if @sync_code_owner_rule
end
def handle_project
sync_rule
end
def sync_rule
unless approvers_exists?
target.approval_rules.regular.delete_all
return
end
rule = first_or_initialize
rule.update(user_ids: target.approver_ids, group_ids: target.approver_group_ids)
rule
end
def target
strong_memoize(:target) do
case @target_type
when 'MergeRequest'
MergeRequest.find_by(id: @target_id)
when 'Project'
Project.find_by(id: @target_id)
end
end
end
def first_or_initialize
rule = target.approval_rules.regular.first_or_initialize
unless rule.persisted?
rule.name ||= ApprovalRuleLike::DEFAULT_NAME
rule.approvals_required = [target.approvals_required, ApprovalRuleLike::APPROVALS_REQUIRED_MAX].min
rule.save!
end
rule
end
def approvers_exists?
target.approver_ids.any? || target.approver_group_ids.any?
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class MoveEpicIssuesAfterEpics
class EpicIssue < ActiveRecord::Base
self.table_name = 'epic_issues'
end
class Epic < ActiveRecord::Base
self.table_name = 'epics'
end
def perform(start_id, stop_id)
maximum_epic_position = Epic.maximum(:relative_position)
return unless maximum_epic_position
max_position = Gitlab::Database::MAX_INT_VALUE
delta = ((maximum_epic_position - max_position) / 2.0).abs.ceil
EpicIssue.where(epic_id: start_id..stop_id).where('relative_position < ?', max_position - delta)
.update_all("relative_position = relative_position + #{delta}")
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given merge request IDs range. A _single_ INSERT is issued for the given range.
class PopulateAnyApprovalRuleForMergeRequests
MAX_VALUE = 2**15 - 1
def perform(from_id, to_id)
select_sql =
MergeRequest
.where(merge_request_approval_rules_not_exists_clause)
.where(id: from_id..to_id)
.where('approvals_before_merge <> 0')
.select("id, LEAST(#{MAX_VALUE}, approvals_before_merge), created_at, updated_at, 4, '#{ApprovalRuleLike::ALL_MEMBERS}'")
.to_sql
execute("INSERT INTO approval_merge_request_rules (merge_request_id, approvals_required, created_at, updated_at, rule_type, name) #{select_sql}")
end
private
def merge_request_approval_rules_not_exists_clause
<<~SQL
NOT EXISTS (SELECT 1 FROM approval_merge_request_rules
WHERE approval_merge_request_rules.merge_request_id = merge_requests.id)
SQL
end
def execute(sql)
@connection ||= ActiveRecord::Base.connection
@connection.execute(sql)
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given project IDs range. A _single_ INSERT is issued for the given range.
class PopulateAnyApprovalRuleForProjects
MAX_VALUE = 2**15 - 1
def perform(from_id, to_id)
select_sql =
Project
.where(project_approval_rules_not_exists_clause)
.where(id: from_id..to_id)
.where('approvals_before_merge <> 0')
.select("id, LEAST(#{MAX_VALUE}, approvals_before_merge), created_at, updated_at, #{ApprovalProjectRule.rule_types[:any_approver]}, '#{ApprovalRuleLike::ALL_MEMBERS}'")
.to_sql
execute("INSERT INTO approval_project_rules (project_id, approvals_required, created_at, updated_at, rule_type, name) #{select_sql}")
end
private
def project_approval_rules_not_exists_clause
<<~SQL
NOT EXISTS (SELECT 1 FROM approval_project_rules
WHERE approval_project_rules.project_id = projects.id)
SQL
end
def execute(sql)
@connection ||= ActiveRecord::Base.connection
@connection.execute(sql)
end
end
end
end
# frozen_string_literal: true
# This job is added to fix https://gitlab.com/gitlab-org/gitlab/issues/30229
# It's not used anywhere else.
# Can be removed in GitLab 13.*
module Gitlab
module BackgroundMigration
class PruneOrphanedGeoEvents
BATCH_SIZE = 50_000
RESCHEDULE_DELAY = 5.minutes
EVENT_TABLES = %w[geo_repository_created_events
geo_repository_updated_events
geo_repository_deleted_events
geo_repository_renamed_events
geo_repositories_changed_events
geo_hashed_storage_migrated_events
geo_hashed_storage_attachments_events
geo_lfs_object_deleted_events
geo_job_artifact_deleted_events
geo_upload_deleted_events].freeze
module PrunableEvent
extend ActiveSupport::Concern
include EachBatch
included do
scope :orphans, -> do
where(<<-SQL.squish)
NOT EXISTS (
SELECT 1
FROM geo_event_log
WHERE geo_event_log.#{geo_event_foreign_key} = #{table_name}.id
)
SQL
end
end
class_methods do
def geo_event_foreign_key
table_name.singularize.sub(/^geo_/, '') + '_id'
end
def delete_batch_of_orphans!
deleted = where(id: orphans.limit(BATCH_SIZE)).delete_all
vacuum! if deleted.positive?
deleted
end
def vacuum!
connection.execute("VACUUM #{table_name}")
rescue ActiveRecord::StatementInvalid => e
# ignore timeout, auto-vacuum will take care of it
raise unless e.message =~ /statement timeout/i
end
end
end
def perform(table_name = EVENT_TABLES.first)
return if Gitlab::Database.read_only?
deleted_rows = prune_orphaned_rows(table_name)
table_name = next_table(table_name) if deleted_rows.zero?
BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name.demodulize, table_name) if table_name
end
def prune_orphaned_rows(table)
event_model(table).delete_batch_of_orphans!
end
def event_model(table)
Class.new(ActiveRecord::Base) do
include PrunableEvent
self.table_name = table
end
end
def next_table(table_name)
return if EVENT_TABLES.last == table_name
index = EVENT_TABLES.index(table_name)
return unless index
EVENT_TABLES[index + 1]
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class UpdateAuthorizedKeysFileSince
include Gitlab::ShellAdapter
class Key < ActiveRecord::Base
self.table_name = 'keys'
def shell_id
"key-#{id}"
end
end
delegate :remove_keys_not_found_in_db, to: :gitlab_shell
def perform(cutoff_datetime)
add_keys_since(cutoff_datetime)
remove_keys_not_found_in_db
end
def add_keys_since(cutoff_datetime)
start_key = Key.select(:id).where("created_at >= ?", cutoff_datetime).order('id ASC').take
if start_key
batch_add_keys_in_db_starting_from(start_key.id)
end
end
# Not added to Gitlab::Shell because I don't expect this to be used again
def batch_add_keys_in_db_starting_from(start_id)
Rails.logger.info("Adding all keys starting from ID: #{start_id}") # rubocop:disable Gitlab/RailsLogger
::Key.find_in_batches(start: start_id, batch_size: 1000) do |keys|
gitlab_shell.batch_add_keys(keys)
end
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
class UpdateVulnerabilityConfidence
class Occurrence < ActiveRecord::Base
include ::EachBatch
self.table_name = 'vulnerability_occurrences'
REPORT_TYPES = {
container_scanning: 2
}.freeze
CONFIDENCE_LEVELS = {
unknown: 2,
medium: 5
}.freeze
enum confidences: CONFIDENCE_LEVELS
enum report_type: REPORT_TYPES
def self.container_scanning_reports_with_medium_confidence
where(report_type: self.report_types[:container_scanning], confidence: self.confidences[:medium])
end
end
def perform(start_id, stop_id)
Occurrence.container_scanning_reports_with_medium_confidence
.where(id: start_id..stop_id)
.update_all(confidence: Occurrence.confidences[:unknown])
end
end
end
end
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class BackfillVersionDataFromGitaly
def perform(issue_id)
end
end
end
end
Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly.prepend_if_ee('EE::Gitlab::BackgroundMigration::BackfillVersionDataFromGitaly')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class GenerateGitlabSubscriptions
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::GenerateGitlabSubscriptions.prepend_if_ee('EE::Gitlab::BackgroundMigration::GenerateGitlabSubscriptions')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class MigrateApproverToApprovalRules
# @param target_type [String] class of target, either 'MergeRequest' or 'Project'
# @param target_id [Integer] id of target
def perform(target_type, target_id, sync_code_owner_rule: true)
end
end
end
end
Gitlab::BackgroundMigration::MigrateApproverToApprovalRules.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRules')
...@@ -2,22 +2,12 @@ ...@@ -2,22 +2,12 @@
module Gitlab module Gitlab
module BackgroundMigration module BackgroundMigration
# rubocop: disable Style/Documentation
class MigrateApproverToApprovalRulesCheckProgress class MigrateApproverToApprovalRulesCheckProgress
RESCHEDULE_DELAY = 1.day
def perform def perform
if remaining?
BackgroundMigrationWorker.perform_in(RESCHEDULE_DELAY, self.class.name)
else
Feature.enable(:approval_rule)
end
end
private
def remaining?
Gitlab::BackgroundMigration.exists?('MigrateApproverToApprovalRulesInBatch')
end end
end end
end end
end end
Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesCheckProgress')
...@@ -2,17 +2,12 @@ ...@@ -2,17 +2,12 @@
module Gitlab module Gitlab
module BackgroundMigration module BackgroundMigration
# rubocop: disable Style/Documentation
class MigrateApproverToApprovalRulesInBatch class MigrateApproverToApprovalRulesInBatch
class MergeRequest < ActiveRecord::Base
self.table_name = 'merge_requests'
end
def perform(start_id, end_id) def perform(start_id, end_id)
merge_request_ids = MergeRequest.where('id >= ? AND id <= ?', start_id, end_id).pluck(:id)
merge_request_ids.each do |merge_request_id|
MigrateApproverToApprovalRules.new.perform('MergeRequest', merge_request_id)
end
end end
end end
end end
end end
Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateApproverToApprovalRulesInBatch')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class MoveEpicIssuesAfterEpics
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics.prepend_if_ee('EE::Gitlab::BackgroundMigration::MoveEpicIssuesAfterEpics')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given merge request IDs range. A _single_ INSERT is issued for the given range.
class PopulateAnyApprovalRuleForMergeRequests
def perform(from_id, to_id)
end
end
end
end
Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForMergeRequests')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# This background migration creates any approver rule records according
# to the given project IDs range. A _single_ INSERT is issued for the given range.
class PopulateAnyApprovalRuleForProjects
def perform(from_id, to_id)
end
end
end
end
Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateAnyApprovalRuleForProjects')
# frozen_string_literal: true
#
# rubocop:disable Style/Documentation
# This job is added to fix https://gitlab.com/gitlab-org/gitlab/issues/30229
# It's not used anywhere else.
# Can be removed in GitLab 13.*
module Gitlab
module BackgroundMigration
class PruneOrphanedGeoEvents
def perform(table_name)
end
end
end
end
Gitlab::BackgroundMigration::PruneOrphanedGeoEvents.prepend_if_ee('EE::Gitlab::BackgroundMigration::PruneOrphanedGeoEvents')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class UpdateAuthorizedKeysFileSince
def perform(cutoff_datetime)
end
end
end
end
Gitlab::BackgroundMigration::UpdateAuthorizedKeysFileSince.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateAuthorizedKeysFileSince')
# frozen_string_literal: true
module Gitlab
module BackgroundMigration
# rubocop: disable Style/Documentation
class UpdateVulnerabilityConfidence
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilityConfidence')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment