Skip to content

G
gitlab-ce
Commit 74e1fd38 authored by Michael Kozono's avatar Michael Kozono
Browse files
Options

Merge branch '223249-remove-fdw-related-code-for-projects-and-wiokis' into 'master' 

Remove FDW related finders to sync/verify projects and wiki

See merge request gitlab-org/gitlab!37850
parents ad89beca 1c5e99ac
Hide whitespace changes
Inline Side-by-side
Showing with 63 additions and 1628 deletions
ee/app/finders/geo/project_registry_mismatch_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving project registries that checksum mismatch
# scoped to a type (repository or wiki) using FDW queries.
#
# Basic usage:
#
# Geo::ProjectRegistryMismatchFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class ProjectRegistryMismatchFinder
def initialize(current_node:, type:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@type = type.to_s.to_sym
end
def execute
current_node.project_registries.mismatch(type)
end
private
attr_reader :current_node, :type
end
end
ee/app/finders/geo/project_registry_pending_verification_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving project registries that that need a repository or
# wiki verification where projects belong to the specific shard using
# FDW queries.
#
# Basic usage:
#
# Geo::ProjectRegistryPendingVerificationFinder
# .new(current_node: Gitlab::Geo.current_node, shard_name: 'default', batch_size: 1000)
# .execute.
module Geo
class ProjectRegistryPendingVerificationFinder
def initialize(current_node:, shard_name:, batch_size:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@shard_name = shard_name
@batch_size = batch_size
end
# rubocop:disable CodeReuse/ActiveRecord
def execute
return Geo::ProjectRegistry.none unless valid_shard?
Gitlab::Geo::Fdw::ProjectRegistryQueryBuilder
.new(current_node.project_registries)
.registries_pending_verification
.within_shards(shard_name)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
private
attr_reader :current_node, :shard_name, :batch_size
def valid_shard?
return true unless current_node.selective_sync_by_shards?
current_node.selective_sync_shards.include?(shard_name)
end
end
end
ee/app/finders/geo/project_registry_retrying_verification_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving project registries that are retrying verification
# scoped to a type (repository or wiki) using FDW queries.
#
# Basic usage:
#
# Geo::ProjectRegistryRetryingVerificationFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class ProjectRegistryRetryingVerificationFinder
def initialize(current_node:, type:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@type = type.to_s.to_sym
end
def execute
current_node.project_registries.retrying_verification(type)
end
private
attr_reader :current_node, :type
end
end
ee/app/finders/geo/project_registry_sync_failed_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving project registries that synchronization have
# failed scoped to a type (repository or wiki) using FDW queries.
#
# Basic usage:
#
# Geo::ProjectRegistrySyncFailedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class ProjectRegistrySyncFailedFinder
def initialize(current_node:, type:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@type = type.to_s.to_sym
end
def execute
current_node.project_registries.sync_failed(type)
end
private
attr_reader :current_node, :type
end
end
ee/app/finders/geo/project_registry_synced_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving project registries that have been synced
# scoped to a type (repository or wiki) using FDW queries.
#
# Basic usage:
#
# Geo::ProjectRegistrySyncedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class ProjectRegistrySyncedFinder
def initialize(current_node:, type:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@type = type.to_sym
end
def execute
current_node.project_registries.synced(type)
end
private
attr_reader :current_node, :type
end
end
ee/app/finders/geo/project_registry_verification_failed_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving project registries that have verification failed
# scoped to a type (repository or wiki) using FDW queries.
#
# Basic usage:
#
# Geo::ProjectRegistryVerificationFailedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class ProjectRegistryVerificationFailedFinder
def initialize(current_node:, type:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@type = type.to_s.to_sym
end
def execute
current_node.project_registries.verification_failed(type)
end
private
attr_reader :current_node, :type
end
end
ee/app/finders/geo/project_registry_verified_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving project registries that have verification succeeded
# scoped to a type (repository or wiki) using FDW queries.
#
# Basic usage:
#
# Geo::ProjectRegistryVerifiedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class ProjectRegistryVerifiedFinder
def initialize(current_node:, type:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@type = type.to_sym
end
def execute
current_node.project_registries.verified(type)
end
private
attr_reader :current_node, :type
end
end
ee/app/finders/geo/project_unsynced_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving unsynced projects that belong to a specific
# shard using FDW queries.
#
# Basic usage:
#
# Geo::ProjectUnsyncedFinder
# .new(current_node: Gitlab::Geo.current_node, shard_name: 'default', batch_size: 1000)
# .execute.
module Geo
class ProjectUnsyncedFinder
def initialize(current_node:, shard_name:, batch_size: nil)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@shard_name = shard_name
@batch_size = batch_size
end
# rubocop:disable CodeReuse/ActiveRecord
def execute
return Geo::Fdw::Project.none unless valid_shard?
relation = projects.missing_project_registry.within_shards(shard_name)
relation = relation.limit(batch_size) unless batch_size.nil?
relation
end
# rubocop:enable CodeReuse/ActiveRecord
private
attr_reader :current_node, :shard_name, :batch_size
def projects
return Geo::Fdw::Project.all if current_node.selective_sync_by_shards?
current_node.projects
end
def valid_shard?
return true unless current_node.selective_sync_by_shards?
current_node.selective_sync_shards.include?(shard_name)
end
end
end
ee/app/finders/geo/project_updated_recently_finder.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
# Finder for retrieving projects updated recently that belong to a specific
# shard using FDW queries.
#
# Basic usage:
#
# Geo::ProjectUpdatedRecentlyFinder
# .new(current_node: Gitlab::Geo.current_node, shard_name: 'default', batch_size: 1000)
# .execute.
module Geo
class ProjectUpdatedRecentlyFinder
def initialize(current_node:, shard_name:, batch_size: nil)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
@shard_name = shard_name
@batch_size = batch_size
end
# rubocop:disable CodeReuse/ActiveRecord
def execute
return Geo::Fdw::Project.none unless valid_shard?
relation = projects.recently_updated.within_shards(shard_name)
relation = relation.limit(batch_size) unless batch_size.nil?
relation
end
# rubocop:enable CodeReuse/ActiveRecord
private
attr_reader :current_node, :shard_name, :batch_size
def projects
return Geo::Fdw::Project.all if current_node.selective_sync_by_shards?
current_node.projects
end
def valid_shard?
return true unless current_node.selective_sync_by_shards?
current_node.selective_sync_shards.include?(shard_name)
end
end
end
ee/app/workers/all_queues.yml
View file @ 74e1fd38
......@@ -475,14 +475,6 @@
:weight: 1
:idempotent:
:tags: []
- :name: geo:geo_secondary_repository_backfill
:feature_category: :geo_replication
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent:
:tags: []
- :name: jira_connect:jira_connect_sync_branch
:feature_category: :integrations
:has_external_dependencies:
......
ee/app/workers/concerns/geo/secondary/backfill_worker.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
module Geo
module Secondary
module BackfillWorker
extend ActiveSupport::Concern
LEASE_TIMEOUT = 60.minutes
RUN_TIME = 60.minutes.to_i
included do
prepend ShadowMethods
include ApplicationWorker
include ExclusiveLeaseGuard
include GeoQueue
include ::Gitlab::Geo::LogHelpers
include ::Gitlab::Utils::StrongMemoize
sidekiq_options retry: false
loggable_arguments 0
attr_reader :shard_name, :start_time, :loops
end
module ShadowMethods
def lease_key
@lease_key ||= "#{self.class.name.underscore}:shard:#{shard_name}"
end
def lease_timeout
LEASE_TIMEOUT
end
end
def initialize
@scheduled_jobs = []
@loops = 0
end
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def perform(shard_name)
@shard_name = shard_name
@start_time = Time.now.utc
return unless healthy_node?
try_obtain_lease do
schedule_jobs
end
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
private
def base_log_data(message)
super(message).merge(worker_metadata)
end
def healthy_node?
unless Gitlab::Geo.geo_database_configured?
log_info('Geo database not configured')
return false
end
unless Gitlab::Geo.secondary?
log_info('Current node not a secondary')
return false
end
unless Gitlab::ShardHealthCache.healthy_shard?(shard_name)
log_info("Shard (#{shard_name}) is not healthy")
return false
end
true
end
# rubocop:disable Gitlab/ModuleWithInstanceVariables
def node_enabled?
# Only check every minute to avoid polling the DB excessively
unless @last_enabled_check.present? && @last_enabled_check > 1.minute.ago
@last_enabled_check = Time.now.utc
clear_memoization(:current_node_enabled)
end
strong_memoize(:current_node_enabled) do
Gitlab::Geo.current_node_enabled?
end
end
# rubocop:enable Gitlab/ModuleWithInstanceVariables
def run_time
RUN_TIME
end
def over_capacity?
false
end
def over_time?
(Time.now.utc - start_time) >= run_time
end
def worker_metadata
{ shard: shard_name }
end
end
end
end
ee/app/workers/geo/repository_sync_worker.rb
View file @ 74e1fd38
......@@ -3,12 +3,7 @@
module Geo
class RepositorySyncWorker < Geo::Scheduler::Secondary::PerShardSchedulerWorker # rubocop:disable Scalability/IdempotentWorker
def schedule_job(shard_name)
if ::Feature.enabled?(:geo_streaming_results_repository_sync)
Geo::Secondary::RepositoryBackfillWorker.perform_async(shard_name)
else
Geo::RepositoryShardSyncWorker.perform_async(shard_name)
end
Geo::RepositoryShardSyncWorker.perform_async(shard_name)
Geo::DesignRepositoryShardSyncWorker.perform_async(shard_name)
end
end
......
ee/app/workers/geo/secondary/repository_backfill_worker.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
module Geo
module Secondary
class RepositoryBackfillWorker # rubocop:disable Scalability/IdempotentWorker
include Geo::Secondary::BackfillWorker
private
attr_reader :scheduled_jobs
def connection
strong_memoize(:connection) { Geo::TrackingBase.connection.raw_connection }
end
def max_capacity
# If we don't have a count, that means that for some reason
# RepositorySyncWorker stopped running/updating the cache. We might
# be trying to shut down Geo while this job may still be running.
healthy_count = healthy_shard_count
return 0 unless healthy_count > 0
capacity_per_shard = Gitlab::Geo.current_node.repos_max_capacity / healthy_count
[1, capacity_per_shard.to_i].max
end
def healthy_shard_count
Gitlab::ShardHealthCache.healthy_shard_count.to_i
end
def over_capacity?
scheduled_jobs.size >= max_capacity
end
def schedule_jobs
log_info('Repository backfilling started')
reason = :unknown
begin
connection.send_query("#{projects_ids_unsynced.to_sql};#{project_ids_updated_recently.to_sql}")
connection.set_single_row_mode
reason = loop do
break :node_disabled unless node_enabled?
break :over_time if over_time?
break :lease_lost unless renew_lease!
update_jobs_in_progress
if over_capacity?
sleep(1)
else
# This will stream the results one by one
# until there are no more results to fetch.
result = connection.get_result
break :complete if result.nil?
result.check
result.each do |row|
schedule_job(row['id'])
end
end
end
rescue => error
reason = :error
log_error('Repository backfilling error', error)
raise error
ensure
log_info('Repository backfilling finished', total_loops: loops, duration: Time.now.utc - start_time, reason: reason)
end
end
def schedule_job(project_id)
job_id = Geo::ProjectSyncWorker.perform_async(project_id, sync_repository: true, sync_wiki: true)
if job_id
@scheduled_jobs << { job_id: job_id }
log_info("Repository scheduled for backfilling", project_id: project_id, job_id: job_id)
else
log_info("Repository could not be scheduled for backfilling", project_id: project_id)
end
end
def scheduled_job_ids
scheduled_jobs.map { |data| data[:job_id] }
end
def update_jobs_in_progress
job_ids = scheduled_job_ids
return if job_ids.empty?
# SidekiqStatus returns an array of booleans: true if the job is still running, false otherwise.
# For each entry, first use `zip` to make { job_id: 123 } -> [ { job_id: 123 }, bool ]
# Next, filter out the jobs that have completed.
@scheduled_jobs = Gitlab::SidekiqStatus.job_status(scheduled_job_ids).then do |status|
@scheduled_jobs.zip(status).map { |(job, running)| job if running }.compact
end
end
# rubocop: disable CodeReuse/ActiveRecord
def projects_ids_unsynced
Geo::ProjectUnsyncedFinder
.new(current_node: Gitlab::Geo.current_node, shard_name: shard_name)
.execute
.reorder(last_repository_updated_at: :desc)
.select(:id)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def project_ids_updated_recently
Geo::ProjectUpdatedRecentlyFinder
.new(current_node: Gitlab::Geo.current_node, shard_name: shard_name)
.execute
.order('project_registry.last_repository_synced_at ASC NULLS FIRST, projects.last_repository_updated_at ASC')
.select(:id)
end
# rubocop: enable CodeReuse/ActiveRecord
end
end
end
ee/lib/api/geo_nodes.rb
View file @ 74e1fd38
......@@ -106,18 +106,19 @@ module API
not_found!('Geo node not found') unless Gitlab::Geo.current_node
forbidden!('Failures can only be requested from a secondary node') unless Gitlab::Geo.current_node.secondary?
finder_klass = case params[:failure_type]
when 'sync'
::Geo::ProjectRegistrySyncFailedFinder
when 'verification'
::Geo::ProjectRegistryVerificationFailedFinder
when 'checksum_mismatch'
::Geo::ProjectRegistryMismatchFinder
else
not_found!('Failure type unknown')
end
project_registries = finder_klass.new(current_node: Gitlab::Geo.current_node, type: params[:type]).execute
type = params[:type].to_s.to_sym
project_registries =
case params[:failure_type]
when 'sync'
::Geo::ProjectRegistry.sync_failed(type)
when 'verification'
::Geo::ProjectRegistry.verification_failed(type)
when 'checksum_mismatch'
::Geo::ProjectRegistry.mismatch(type)
else
not_found!('Failure type unknown')
end
present paginate(project_registries), with: ::GeoProjectRegistryEntity
end
......
ee/spec/finders/geo/project_registry_mismatch_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectRegistryMismatchFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_mismatch) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_checksum_mismatch, project: project_1) }
let!(:registry_repository_mismatch) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_verified, project: project_2) }
let!(:registry_wiki_mismatch) { create(:geo_project_registry, :repository_verified, :wiki_checksum_mismatch, project: project_3) }
let!(:registry_wiki_mismatch_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_checksum_mismatch, project: project_4) }
let!(:registry_repository_mismatch_broken_shard) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_verified, project: project_5) }
let!(:registry_verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all mismatch registries' do
expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_repository_mismatch_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns mismatch registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch)
end
end
context 'with selective sync by shard' do
it 'returns mismatch registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_mismatch_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all mismatch registries' do
expect(subject.execute).to contain_exactly(registry_mismatch, registry_wiki_mismatch, registry_wiki_mismatch_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns mismatch registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_mismatch, registry_wiki_mismatch)
end
end
context 'with selective sync by shard' do
it 'returns mismatch registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_wiki_mismatch_broken_shard)
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns all mismatch registries' do
expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_wiki_mismatch, registry_repository_mismatch_broken_shard, registry_wiki_mismatch_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns all mismatch registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_wiki_mismatch)
end
end
context 'with selective sync by shard' do
it 'returns all mismatch registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_mismatch_broken_shard, registry_wiki_mismatch_broken_shard)
end
end
end
end
end
ee/spec/finders/geo/project_registry_pending_verification_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectRegistryPendingVerificationFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
subject { described_class.new(current_node: node, shard_name: 'default', batch_size: 100) }
it 'does not return registries that are verified on primary and secondary' do
project_verified = create(:repository_state, :repository_verified, :wiki_verified).project
repository_verified = create(:repository_state, :repository_verified).project
wiki_verified = create(:repository_state, :wiki_verified).project
create(:geo_project_registry, :repository_verified, :wiki_verified, project: project_verified)
create(:geo_project_registry, :repository_verified, project: repository_verified)
create(:geo_project_registry, :wiki_verified, project: wiki_verified)
expect(subject.execute).to be_empty
end
it 'does not return registries that were unverified/outdated on primary' do
project_unverified_primary = create(:project)
project_outdated_primary = create(:repository_state, :repository_outdated, :wiki_outdated).project
repository_outdated_primary = create(:repository_state, :repository_outdated, :wiki_verified).project
wiki_outdated_primary = create(:repository_state, :repository_verified, :wiki_outdated).project
create(:geo_project_registry, project: project_unverified_primary)
create(:geo_project_registry, :repository_verification_outdated, :wiki_verification_outdated, project: project_outdated_primary)
create(:geo_project_registry, :repository_verified, :wiki_verified, project: repository_outdated_primary)
create(:geo_project_registry, :repository_verified, :wiki_verified, project: wiki_outdated_primary)
expect(subject.execute).to be_empty
end
it 'returns registries that were unverified/outdated on secondary' do
project_unverified_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
project_outdated_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
repository_outdated_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
wiki_outdated_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
registry_unverified_secondary = create(:geo_project_registry, :synced, project: project_unverified_secondary)
registry_outdated_secondary = create(:geo_project_registry, :synced, :repository_verification_outdated, :wiki_verification_outdated, project: project_outdated_secondary)
registry_repository_outdated_secondary = create(:geo_project_registry, :synced, :repository_verification_outdated, :wiki_verified, project: repository_outdated_secondary)
registry_wiki_outdated_secondary = create(:geo_project_registry, :synced, :repository_verified, :wiki_verification_outdated, project: wiki_outdated_secondary)
expect(subject.execute)
.to contain_exactly(
registry_unverified_secondary,
registry_outdated_secondary,
registry_repository_outdated_secondary,
registry_wiki_outdated_secondary
)
end
it 'does not return registries that failed on primary' do
verification_failed_primary = create(:repository_state, :repository_failed, :wiki_failed).project
create(:geo_project_registry, project: verification_failed_primary)
expect(subject.execute).to be_empty
end
it 'returns registries where one failed and one verified on the primary' do
verification_failed_primary = create(:repository_state, :repository_failed, :wiki_failed).project
repository_failed_primary = create(:repository_state, :repository_failed, :wiki_verified).project
wiki_failed_primary = create(:repository_state, :repository_verified, :wiki_failed).project
create(:geo_project_registry, :synced, project: verification_failed_primary)
registry_repository_failed_primary = create(:geo_project_registry, :synced, project: repository_failed_primary)
registry_wiki_failed_primary = create(:geo_project_registry, :synced, project: wiki_failed_primary)
expect(subject.execute)
.to contain_exactly(
registry_repository_failed_primary,
registry_wiki_failed_primary
)
end
it 'does not return registries where verification failed on secondary' do
verification_failed_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
repository_failed_secondary = create(:repository_state, :repository_verified).project
wiki_failed_secondary = create(:repository_state, :wiki_verified).project
create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed, project: verification_failed_secondary)
create(:geo_project_registry, :repository_verification_failed, project: repository_failed_secondary)
create(:geo_project_registry, :wiki_verification_failed, project: wiki_failed_secondary)
expect(subject.execute).to be_empty
end
it 'does not return registries when the repo needs to be resynced' do
project_verified = create(:repository_state, :repository_verified).project
create(:geo_project_registry, :repository_sync_failed, project: project_verified)
expect(subject.execute).to be_empty
end
it 'does not return registries when the wiki needs to be resynced' do
project_verified = create(:repository_state, :wiki_verified).project
create(:geo_project_registry, :wiki_sync_failed, project: project_verified)
expect(subject.execute).to be_empty
end
it 'does not return registries when the repository is missing on primary' do
project_verified = create(:repository_state, :repository_verified).project
create(:geo_project_registry, :synced, project: project_verified, repository_missing_on_primary: true)
expect(subject.execute).to be_empty
end
it 'does not return registries when the wiki is missing on primary' do
project_verified = create(:repository_state, :wiki_verified).project
create(:geo_project_registry, :synced, project: project_verified, wiki_missing_on_primary: true)
expect(subject.execute).to be_empty
end
it 'does not return registries where projects belongs to other shards' do
project_broken_storage = create(:project, :broken_storage)
create(:repository_state, :repository_verified, :wiki_verified, project: project_broken_storage)
create(:geo_project_registry, :synced, project: project_broken_storage)
expect(subject.execute).to be_empty
end
context 'with selective sync by namespace' do
it 'returns registries where projects belongs to the namespaces' do
group_1 = create(:group)
group_2 = create(:group)
nested_group_1 = create(:group, parent: group_1)
project_1 = create(:project, group: group_1)
project_2 = create(:project, group: nested_group_1)
project_3 = create(:project, group: group_2)
create(:repository_state, :repository_verified, :wiki_verified, project: project_1)
create(:repository_state, :repository_verified, :wiki_verified, project: project_2)
create(:repository_state, :repository_verified, :wiki_verified, project: project_3)
registry_unverified_secondary = create(:geo_project_registry, :synced, project: project_1)
registry_outdated_secondary = create(:geo_project_registry, :synced, :repository_verification_outdated, :wiki_verification_outdated, project: project_2)
create(:geo_project_registry, :synced, :repository_verification_outdated, :wiki_verified, project: project_3)
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute)
.to contain_exactly(
registry_unverified_secondary,
registry_outdated_secondary
)
end
end
context 'with selective sync by shard' do
let(:project_broken_storage) { create(:project, :broken_storage) }
let!(:repository_state_project_broken_storage) { create(:repository_state, :repository_verified, :wiki_verified, project: project_broken_storage) }
let!(:registry_repository_broken_shard) { create(:geo_project_registry, :synced, project: project_broken_storage) }
let(:project) { create(:project) }
let!(:project_unverified_secondary) { create(:repository_state, :repository_verified, :wiki_verified, project: project) }
let!(:registry_unverified_secondary) { create(:geo_project_registry, :synced, project: project) }
before do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'does not return registries when selected shards to sync does not include the shard_name' do
subject = described_class.new(current_node: node, shard_name: 'default', batch_size: 100)
expect(subject.execute).to be_empty
end
it 'returns registries where projects belongs to the shards' do
subject = described_class.new(current_node: node, shard_name: 'broken', batch_size: 100)
expect(subject.execute).to contain_exactly(registry_repository_broken_shard)
end
end
end
end
ee/spec/finders/geo/project_registry_retrying_verification_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectRegistryRetryingVerificationFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:retrying_verification) { create(:geo_project_registry, :repository_retrying_verification, :wiki_retrying_verification, project: project_1) }
let!(:repository_retrying_verification) { create(:geo_project_registry, :repository_retrying_verification, :wiki_verified, project: project_2) }
let!(:wiki_retrying_verification) { create(:geo_project_registry, :repository_verified, :wiki_retrying_verification, project: project_3) }
let!(:wiki_retrying_verification_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_retrying_verification, project: project_4) }
let!(:repository_retrying_verification_broken_shard) { create(:geo_project_registry, :repository_retrying_verification, :wiki_verified, project: project_5) }
let!(:verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all registries retrying verification' do
expect(subject.execute).to contain_exactly(retrying_verification, repository_retrying_verification, repository_retrying_verification_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns registries retrying verification where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(retrying_verification, repository_retrying_verification)
end
end
context 'with selective sync by shard' do
it 'returns registries retrying verification where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(repository_retrying_verification_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all registries retrying verification' do
expect(subject.execute).to contain_exactly(retrying_verification, wiki_retrying_verification, wiki_retrying_verification_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns registries retrying verification where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(retrying_verification, wiki_retrying_verification)
end
end
context 'with selective sync by shard' do
it 'returns registries retrying verification where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(wiki_retrying_verification_broken_shard)
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
context 'with selective sync by namespace' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
context 'with selective sync by shard' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
end
end
end
ee/spec/finders/geo/project_registry_sync_failed_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectRegistrySyncFailedFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_failed) { create(:geo_project_registry, :sync_failed, project: project_1) }
let!(:registry_repository_failed) { create(:geo_project_registry, :synced, :repository_sync_failed, project: project_2) }
let!(:registry_wiki_failed) { create(:geo_project_registry, :synced, :wiki_sync_failed, project: project_3) }
let!(:registry_wiki_failed_broken_shard) { create(:geo_project_registry, :synced, :wiki_sync_failed, project: project_4) }
let!(:registry_repository_failed_broken_shard) { create(:geo_project_registry, :synced, :repository_sync_failed, project: project_5) }
let!(:registry_synced) { create(:geo_project_registry, :synced) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_repository_failed_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_failed, registry_repository_failed])
end
end
context 'with selective sync by shard' do
it 'returns failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_repository_failed_broken_shard])
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to match_array([registry_failed, registry_wiki_failed, registry_wiki_failed_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_failed, registry_wiki_failed])
end
end
context 'with selective sync by shard' do
it 'returns failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_wiki_failed_broken_shard])
end
end
end
context 'with no type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_wiki_failed, registry_repository_failed_broken_shard, registry_wiki_failed_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns all failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_wiki_failed])
end
end
context 'with selective sync by shard' do
it 'returns all failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_repository_failed_broken_shard, registry_wiki_failed_broken_shard])
end
end
end
end
end
ee/spec/finders/geo/project_registry_synced_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectRegistrySyncedFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_synced) { create(:geo_project_registry, :synced, project: project_1) }
let!(:registry_repository_dirty) { create(:geo_project_registry, :synced, :repository_dirty, project: project_2) }
let!(:registry_wiki_dirty) { create(:geo_project_registry, :synced, :wiki_dirty, project: project_3) }
let!(:registry_wiki_dirty_broken_shard) { create(:geo_project_registry, :synced, :wiki_dirty, project: project_4) }
let!(:registry_repository_dirty_broken_shard) { create(:geo_project_registry, :synced, :repository_dirty, project: project_5) }
let!(:registry_sync_failed) { create(:geo_project_registry, :sync_failed) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all synced registries' do
expect(subject.execute).to match_array([registry_synced, registry_wiki_dirty, registry_wiki_dirty_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns synced registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_synced, registry_wiki_dirty])
end
end
context 'with selective sync by shard' do
it 'returns synced registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_wiki_dirty_broken_shard])
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all synced registries' do
expect(subject.execute).to match_array([registry_synced, registry_repository_dirty, registry_repository_dirty_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns synced registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_synced, registry_repository_dirty])
end
end
context 'with selective sync by shard' do
it 'returns synced registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_repository_dirty_broken_shard])
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
end
end
ee/spec/finders/geo/project_registry_verification_failed_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectRegistryVerificationFailedFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed, project: project_1) }
let!(:registry_repository_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_2) }
let!(:registry_wiki_verification_failed) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_3) }
let!(:registry_wiki_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_4) }
let!(:registry_repository_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_5) }
let!(:registry_verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_repository_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_verification_failed_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_wiki_verification_failed, registry_wiki_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_wiki_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_wiki_verification_failed_broken_shard)
end
end
end
context 'with no type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_wiki_verification_failed, registry_repository_verification_failed_broken_shard, registry_wiki_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns all failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_wiki_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns all failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_verification_failed_broken_shard, registry_wiki_verification_failed_broken_shard)
end
end
end
end
end
ee/spec/finders/geo/project_registry_verified_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectRegistryVerifiedFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_verified) { create(:geo_project_registry, :repository_verified, :wiki_verified, project: project_1) }
let!(:registry_repository_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_2) }
let!(:registry_repository_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_5) }
let!(:registry_wiki_verification_failed) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_3) }
let!(:registry_wiki_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_4) }
let!(:registry_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all verified registries' do
expect(subject.execute).to contain_exactly(registry_verified, registry_wiki_verification_failed, registry_wiki_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns verified registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verified, registry_wiki_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns verified registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_wiki_verification_failed_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all verified registries' do
expect(subject.execute).to contain_exactly(registry_verified, registry_repository_verification_failed, registry_repository_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns verified registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verified, registry_repository_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns verified registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_verification_failed_broken_shard)
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
end
end
ee/spec/finders/geo/project_unsynced_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectUnsyncedFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let!(:project_1) { create(:project, group: group_1) }
let!(:project_2) { create(:project, group: nested_group_1) }
let!(:project_3) { create(:project, group: group_2) }
let!(:project_4) { create(:project, group: group_1) }
before do
project_4.update_column(:repository_storage, 'foo')
end
subject { described_class.new(current_node: node, shard_name: 'default', batch_size: 100) }
context 'without selective sync' do
it 'returns projects without an entry on the tracking database' do
create(:geo_project_registry, :synced, project: project_2)
expect(subject.execute).to match_ids(project_1, project_3)
end
end
context 'with selective sync by namespace' do
it 'returns projects that belong to the namespaces without an entry on the tracking database' do
create(:geo_project_registry, :synced, project: project_4)
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
expect(subject.execute).to match_ids(project_1, project_2)
end
end
context 'with selective sync by shard' do
before do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['foo'])
end
it 'does not return registries when selected shards to sync does not include the shard_name' do
subject = described_class.new(current_node: node, shard_name: 'default', batch_size: 100)
expect(subject.execute).to be_empty
end
it 'returns projects that belong to the shards without an entry on the tracking database' do
project_5 = create(:project, group: group_1)
project_5.update_column(:repository_storage, 'foo')
create(:geo_project_registry, :synced, project: project_4)
subject = described_class.new(current_node: node, shard_name: 'foo', batch_size: 100)
expect(subject.execute).to match_ids(project_5)
end
end
end
end
ee/spec/finders/geo/project_updated_recently_finder_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::ProjectUpdatedRecentlyFinder, :geo, :geo_fdw do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let!(:project_1) { create(:project, group: group_1) }
let!(:project_2) { create(:project, group: nested_group_1) }
let!(:project_3) { create(:project, group: group_2) }
let!(:project_4) { create(:project, group: group_1) }
before do
project_4.update_column(:repository_storage, 'foo')
create(:geo_project_registry, :synced, :repository_dirty, project: project_1)
create(:geo_project_registry, :synced, :repository_dirty, project: project_2)
create(:geo_project_registry, :synced, project: project_3)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_4)
end
subject { described_class.new(current_node: node, shard_name: 'default', batch_size: 100) }
context 'without selective sync' do
it 'returns projects with a dirty entry on the tracking database' do
expect(subject.execute).to match_ids(project_1, project_2)
end
end
context 'with selective sync by namespace' do
it 'returns projects that belong to the namespaces with a dirty entry on the tracking database' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_ids(project_1, project_2)
end
end
context 'with selective sync by shard' do
before do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['foo'])
end
it 'does not return registries when selected shards to sync does not include the shard_name' do
subject = described_class.new(current_node: node, shard_name: 'default', batch_size: 100)
expect(subject.execute).to be_empty
end
it 'returns projects that belong to the shards with a dirty entry on the tracking database' do
project_5 = create(:project, group: group_1)
project_5.update_column(:repository_storage, 'foo')
create(:geo_project_registry, :synced, project: project_5)
subject = described_class.new(current_node: node, shard_name: 'foo', batch_size: 100)
expect(subject.execute).to match_ids(project_4)
end
end
end
end
ee/spec/workers/geo/repository_sync_worker_spec.rb
View file @ 74e1fd38
......@@ -12,6 +12,7 @@ RSpec.describe Geo::RepositorySyncWorker, :geo, :clean_gitlab_redis_cache do
let!(:unsynced_project) { create(:project) }
let(:healthy_shard_name) { project_in_synced_group.repository.storage }
let(:design_worker) { Geo::DesignRepositoryShardSyncWorker }
let(:repository_worker) { Geo::RepositoryShardSyncWorker }
before do
stub_current_geo_node(secondary)
......@@ -21,91 +22,73 @@ RSpec.describe Geo::RepositorySyncWorker, :geo, :clean_gitlab_redis_cache do
Sidekiq::Testing.inline! { example.run }
end
shared_examples '#perform' do |worker|
context 'additional shards' do
it 'skips backfill for repositories on other shards' do
create(:project, :broken_storage, group: synced_group)
unhealthy_dirty = create(:project, :broken_storage, group: synced_group)
create(:geo_project_registry, :synced, :repository_dirty, project: unhealthy_dirty)
context 'additional shards' do
it 'skips backfill for repositories on other shards' do
create(:project, :broken_storage, group: synced_group)
unhealthy_dirty = create(:project, :broken_storage, group: synced_group)
create(:geo_project_registry, :synced, :repository_dirty, project: unhealthy_dirty)
allow(Gitlab::GitalyClient).to receive(:call) do
raise GRPC::Unavailable.new('No Gitaly available')
end
expect(worker).not_to receive(:perform_async).with('broken')
expect(design_worker).not_to receive(:perform_async).with('broken')
subject.perform
allow(Gitlab::GitalyClient).to receive(:call) do
raise GRPC::Unavailable.new('No Gitaly available')
end
it 'skips backfill for projects on shards excluded by selective sync' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: [healthy_shard_name])
expect(repository_worker).not_to receive(:perform_async).with('broken')
expect(design_worker).not_to receive(:perform_async).with('broken')
# Report both shards as healthy
expect(Gitlab::HealthChecks::GitalyCheck).to receive(:readiness)
.and_return([result(true, healthy_shard_name), result(true, 'broken')])
subject.perform
end
expect(worker).to receive(:perform_async).with('default')
expect(design_worker).to receive(:perform_async).with('default')
expect(worker).not_to receive(:perform_async).with('broken')
expect(design_worker).not_to receive(:perform_async).with('broken')
it 'skips backfill for projects on shards excluded by selective sync' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: [healthy_shard_name])
subject.perform
end
# Report both shards as healthy
expect(Gitlab::HealthChecks::GitalyCheck).to receive(:readiness)
.and_return([result(true, healthy_shard_name), result(true, 'broken')])
it 'skips backfill for projects on missing shards' do
missing_not_synced = create(:project, group: synced_group)
missing_not_synced.update_column(:repository_storage, 'unknown')
missing_dirty = create(:project, group: synced_group)
missing_dirty.update_column(:repository_storage, 'unknown')
expect(repository_worker).to receive(:perform_async).with('default')
expect(design_worker).to receive(:perform_async).with('default')
expect(repository_worker).not_to receive(:perform_async).with('broken')
expect(design_worker).not_to receive(:perform_async).with('broken')
create(:geo_project_registry, :synced, :repository_dirty, project: missing_dirty)
# hide the 'broken' storage for this spec
stub_storage_settings({})
subject.perform
end
expect(worker).to receive(:perform_async).with(project_in_synced_group.repository.storage)
expect(design_worker).to receive(:perform_async).with(project_in_synced_group.repository.storage)
expect(worker).not_to receive(:perform_async).with('unknown')
expect(design_worker).not_to receive(:perform_async).with('unknown')
it 'skips backfill for projects on missing shards' do
missing_not_synced = create(:project, group: synced_group)
missing_not_synced.update_column(:repository_storage, 'unknown')
missing_dirty = create(:project, group: synced_group)
missing_dirty.update_column(:repository_storage, 'unknown')
subject.perform
end
create(:geo_project_registry, :synced, :repository_dirty, project: missing_dirty)
it 'skips backfill for projects with downed Gitaly server' do
create(:project, :broken_storage, group: synced_group)
unhealthy_dirty = create(:project, :broken_storage, group: synced_group)
# hide the 'broken' storage for this spec
stub_storage_settings({})
create(:geo_project_registry, :synced, :repository_dirty, project: unhealthy_dirty)
expect(repository_worker).to receive(:perform_async).with(project_in_synced_group.repository.storage)
expect(design_worker).to receive(:perform_async).with(project_in_synced_group.repository.storage)
expect(repository_worker).not_to receive(:perform_async).with('unknown')
expect(design_worker).not_to receive(:perform_async).with('unknown')
# Report only one healthy shard
expect(Gitlab::HealthChecks::GitalyCheck).to receive(:readiness)
.and_return([result(true, healthy_shard_name), result(false, 'broken')])
subject.perform
end
expect(worker).to receive(:perform_async).with(healthy_shard_name)
expect(design_worker).to receive(:perform_async).with(healthy_shard_name)
expect(worker).not_to receive(:perform_async).with('broken')
expect(design_worker).not_to receive(:perform_async).with('broken')
it 'skips backfill for projects with downed Gitaly server' do
create(:project, :broken_storage, group: synced_group)
unhealthy_dirty = create(:project, :broken_storage, group: synced_group)
subject.perform
end
end
end
create(:geo_project_registry, :synced, :repository_dirty, project: unhealthy_dirty)
context 'when geo_streaming_results_repository_sync flag is enabled', :geo_fdw do
before do
stub_feature_flags(geo_streaming_results_repository_sync: true)
end
# Report only one healthy shard
expect(Gitlab::HealthChecks::GitalyCheck).to receive(:readiness)
.and_return([result(true, healthy_shard_name), result(false, 'broken')])
include_examples '#perform', Geo::Secondary::RepositoryBackfillWorker
end
expect(repository_worker).to receive(:perform_async).with(healthy_shard_name)
expect(design_worker).to receive(:perform_async).with(healthy_shard_name)
expect(repository_worker).not_to receive(:perform_async).with('broken')
expect(design_worker).not_to receive(:perform_async).with('broken')
context 'when geo_streaming_results_repository_sync flag is disabled' do
before do
stub_feature_flags(geo_streaming_results_repository_sync: false)
subject.perform
end
include_examples '#perform', Geo::RepositoryShardSyncWorker
end
def result(success, shard)
......
ee/spec/workers/geo/secondary/repository_backfill_worker_spec.rb deleted 100644 → 0
View file @ ad89beca
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::Secondary::RepositoryBackfillWorker, :geo, :geo_fdw, :clean_gitlab_redis_cache do
include EE::GeoHelpers
let(:primary) { create(:geo_node, :primary) }
let(:secondary) { create(:geo_node, repos_max_capacity: 5) }
let(:shard_name) { Gitlab.config.repositories.storages.each_key.first }
before do
stub_current_geo_node(secondary)
stub_healthy_shards(shard_name)
end
it 'disables Sidekiq retries' do
expect(subject.sidekiq_options_hash).to eq(
'retry' => false,
'queue' => 'geo:geo_secondary_repository_backfill',
'queue_namespace' => :geo
)
end
describe '#perform' do
it 'does not schedule jobs when Geo database is not configured' do
create(:project)
expect(Geo::ProjectSyncWorker).not_to receive(:perform_async)
with_no_geo_database_configured do
subject.perform(shard_name)
end
end
it 'does not schedule jobs when not running on a Geo secondary node' do
stub_current_geo_node(primary)
create(:project)
expect(Geo::ProjectSyncWorker).not_to receive(:perform_async)
subject.perform(shard_name)
end
it 'does not schedule jobs when shard is not healthy' do
stub_healthy_shards([])
create(:project)
expect(Geo::ProjectSyncWorker).not_to receive(:perform_async)
subject.perform(shard_name)
end
it 'does not schedule jobs when the Geo secondary node is disabled' do
stub_node_disabled(secondary)
create(:project)
expect(Geo::ProjectSyncWorker).not_to receive(:perform_async)
subject.perform(shard_name)
end
it 'does not schedule jobs for projects on other shards' do
project = create(:project)
project.update_column(:repository_storage, 'other')
expect(Geo::ProjectSyncWorker).not_to receive(:perform_async)
subject.perform(shard_name)
end
it 'schedules a job for each unsynced project' do
create_list(:project, 2)
expect(Geo::ProjectSyncWorker).to receive(:perform_async).twice.and_return(true)
subject.perform(shard_name)
end
it 'schedules a job for each project where last attempt to sync failed' do
create(:geo_project_registry, :sync_failed)
create(:geo_project_registry, :synced)
expect(Geo::ProjectSyncWorker).to receive(:perform_async).once.and_return(true)
subject.perform(shard_name)
end
it 'schedules a job for each synced project updated recently' do
create(:geo_project_registry, :synced, :repository_dirty)
create(:geo_project_registry, :synced)
create(:geo_project_registry, :synced, :wiki_dirty)
expect(Geo::ProjectSyncWorker).to receive(:perform_async).twice.and_return(true)
subject.perform(shard_name)
end
it 'respects Geo secondary node max capacity per shard' do
stub_healthy_shards([shard_name, 'shard2', 'shard3', 'shard4', 'shard5'])
project_1 = create(:project)
project_2 = create(:project)
allow(Geo::ProjectSyncWorker).to receive(:perform_async).with(project_1.id, anything).and_return('jid-1')
allow(Geo::ProjectSyncWorker).to receive(:perform_async).with(project_2.id, anything).and_return('jid-2')
allow(Gitlab::SidekiqStatus).to receive(:job_status).with(['jid-2']).and_return([true], [false])
allow(Gitlab::SidekiqStatus).to receive(:job_status).with(['jid-1']).and_return([false])
expect(subject).to receive(:sleep).once.and_call_original
subject.perform(shard_name)
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7