Skip to content

G
gitlab-ce
Commit 7a06ed3f authored by Ash McKenzie's avatar Ash McKenzie
Browse files
Options

Merge branch '11922-geo-remove-legacy-queries' into 'master' 

Geo - Remove legacy queries from ProjectRegistryFinder, ExpireUploadsFinder, and RepositoryVerificationFinder

See merge request gitlab-org/gitlab-ee!14195
parents 47592413 54769cf1
Hide whitespace changes
Inline Side-by-side
Showing with 580 additions and 2809 deletions
ee/app/finders/geo/attachment_registry_finder.rb
View file @ 7a06ed3f
......@@ -2,30 +2,36 @@
module Geo
class AttachmentRegistryFinder < FileRegistryFinder
def initialize(current_node:)
@current_node = Geo::Fdw::GeoNode.find(current_node.id)
end
def count_registry
Geo::FileRegistry.attachments.count
end
def count_syncable
syncable.count
end
def count_synced
attachments_synced.count
registries_for_attachments.syncable.merge(Geo::FileRegistry.synced).count
end
def count_failed
attachments_failed.count
registries_for_attachments.syncable.merge(Geo::FileRegistry.failed).count
end
def count_synced_missing_on_primary
attachments_synced_missing_on_primary.count
end
def count_registry
Geo::FileRegistry.attachments.count
registries_for_attachments
.syncable
.merge(Geo::FileRegistry.synced)
.merge(Geo::FileRegistry.missing_on_primary)
.count
end
def syncable
if use_legacy_queries_for_selective_sync?
legacy_finder.syncable
elsif selective_sync?
if selective_sync?
attachments.syncable
else
Upload.syncable
......@@ -44,27 +50,22 @@ module Geo
# @param [Array<Integer>] except_file_ids ids that will be ignored from the query
# rubocop: disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_file_ids: [])
relation =
if use_legacy_queries_for_selective_sync?
legacy_finder.attachments_unsynced(except_file_ids: except_file_ids)
else
attachments_unsynced(except_file_ids: except_file_ids)
end
relation.limit(batch_size)
attachments
.missing_file_registry
.syncable
.id_not_in(except_file_ids)
.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local(batch_size:, except_file_ids: [])
relation =
if use_legacy_queries_for_selective_sync?
legacy_finder.attachments_migrated_local(except_file_ids: except_file_ids)
else
attachments_migrated_local(except_file_ids: except_file_ids)
end
relation.limit(batch_size)
attachments
.inner_join_file_registry
.with_files_stored_remotely
.merge(Geo::FileRegistry.attachments)
.id_not_in(except_file_ids)
.limit(batch_size)
end
# rubocop: enable CodeReuse/ActiveRecord
......@@ -93,74 +94,12 @@ module Geo
private
# rubocop:disable CodeReuse/Finder
def legacy_finder
@legacy_finder ||= Geo::LegacyAttachmentRegistryFinder.new(current_node: current_node)
end
# rubocop:enable CodeReuse/Finder
def fdw_geo_node
@fdw_geo_node ||= Geo::Fdw::GeoNode.find(current_node.id)
end
def registries_for_attachments
if use_legacy_queries_for_selective_sync?
legacy_finder.registries_for_attachments
else
attachments
.inner_join_file_registry
.merge(Geo::FileRegistry.attachments)
end
end
def attachments
fdw_geo_node.attachments
end
def attachments_synced
if use_legacy_queries_for_selective_sync?
legacy_finder.attachments_synced
else
registries_for_attachments
.syncable
.merge(Geo::FileRegistry.synced)
end
end
def attachments_migrated_local(except_file_ids:)
attachments
.inner_join_file_registry
.with_files_stored_remotely
.merge(Geo::FileRegistry.attachments)
.id_not_in(except_file_ids)
end
def attachments_unsynced(except_file_ids:)
attachments
.missing_file_registry
.syncable
.id_not_in(except_file_ids)
current_node.attachments
end
def attachments_failed
if use_legacy_queries_for_selective_sync?
legacy_finder.attachments_failed
else
registries_for_attachments
.syncable
.merge(Geo::FileRegistry.failed)
end
end
def attachments_synced_missing_on_primary
if use_legacy_queries_for_selective_sync?
legacy_finder.attachments_synced_missing_on_primary
else
registries_for_attachments
.syncable
.merge(Geo::FileRegistry.synced)
.merge(Geo::FileRegistry.missing_on_primary)
end
def registries_for_attachments
attachments.inner_join_file_registry.merge(Geo::FileRegistry.attachments)
end
end
end
ee/app/finders/geo/expire_uploads_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
module Geo
class ExpireUploadsFinder
UPLOAD_TYPE = 'file'
def find_project_uploads(project)
if Gitlab::Geo::Fdw.enabled?
Geo::Fdw::Upload.for_model_with_type(project, UPLOAD_TYPE)
else
legacy_find_project_uploads(project)
end
end
def find_file_registries_uploads(project)
if Gitlab::Geo::Fdw.enabled?
Gitlab::Geo::Fdw::UploadRegistryQueryBuilder.new
.for_model(project)
.with_type(UPLOAD_TYPE)
else
legacy_find_file_registries_uploads(project)
end
end
private
# rubocop:disable CodeReuse/ActiveRecord
def legacy_find_file_registries_uploads(project)
upload_ids = Upload.for_model(project).pluck_primary_key
return Geo::FileRegistry.none if upload_ids.empty?
values_sql = upload_ids.map { |id| "(#{id})" }.join(',')
Geo::FileRegistry.joins(<<~SQL)
JOIN (VALUES #{values_sql})
AS uploads (id)
ON uploads.id = file_registry.file_id
AND file_registry.file_type='#{UPLOAD_TYPE}'
SQL
end
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
def legacy_find_project_uploads(project)
file_registry_ids = legacy_find_file_registries_uploads(project).pluck(:file_id)
return Upload.none if file_registry_ids.empty?
values_sql = file_registry_ids.map { |f_id| "(#{f_id})" }.join(',')
Upload.joins(<<~SQL)
JOIN (VALUES #{values_sql})
AS file_registry (file_id)
ON (file_registry.file_id = uploads.id)
SQL
end
# rubocop:enable CodeReuse/ActiveRecord
end
end
ee/app/finders/geo/legacy_attachment_registry_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
module Geo
class LegacyAttachmentRegistryFinder < RegistryFinder
def syncable
attachments.syncable
end
def attachments_synced
legacy_inner_join_registry_ids(
syncable,
Geo::FileRegistry.attachments.synced.pluck_file_key,
Upload
)
end
def attachments_migrated_local(except_file_ids:)
registry_file_ids = Geo::FileRegistry.attachments.pluck_file_key - except_file_ids
legacy_inner_join_registry_ids(
attachments.with_files_stored_remotely,
registry_file_ids,
Upload
)
end
def attachments_unsynced(except_file_ids:)
registry_file_ids = Geo::FileRegistry.attachments.pluck_file_key | except_file_ids
legacy_left_outer_join_registry_ids(
syncable,
registry_file_ids,
Upload
)
end
def attachments_failed
legacy_inner_join_registry_ids(
syncable,
Geo::FileRegistry.attachments.failed.pluck_file_key,
Upload
)
end
def attachments_synced_missing_on_primary
legacy_inner_join_registry_ids(
syncable,
Geo::FileRegistry.attachments.synced.missing_on_primary.pluck_file_key,
Upload
)
end
def registries_for_attachments
return Geo::FileRegistry.attachments unless selective_sync?
legacy_inner_join_registry_ids(
Geo::FileRegistry.attachments,
attachments.pluck_primary_key,
Geo::FileRegistry,
foreign_key: :file_id
)
end
private
def attachments
current_node.attachments
end
end
end
ee/app/finders/geo/legacy_project_registry_mismatch_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving project registries that checksum mismatch
# scoped to a type (repository or wiki) using cross-database joins
# for selective sync.
#
# Basic usage:
#
# Geo::LegacyProjectRegistryMismatchFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class LegacyProjectRegistryMismatchFinder < RegistryFinder
def initialize(current_node: nil, type:)
super(current_node: current_node)
@type = type.to_s.to_sym
end
def execute
legacy_inner_join_registry_ids(
Geo::ProjectRegistry.mismatch(type),
current_node.projects.pluck_primary_key,
Geo::ProjectRegistry,
foreign_key: :project_id
)
end
private
attr_reader :type
end
end
ee/app/finders/geo/legacy_project_registry_pending_verification_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving project registries that need a repository or
# wiki verification where projects belong to the specific shard
# using cross-database joins.
#
# Basic usage:
#
# Geo::LegacyProjectRegistryPendingVerificationFinder
# .new(current_node: Gitlab::Geo.current_node, shard_name: 'default', batch_size: 1000)
# .execute
module Geo
class LegacyProjectRegistryPendingVerificationFinder < RegistryFinder
def initialize(current_node: nil, shard_name:, batch_size:)
super(current_node: current_node)
@shard_name = shard_name
@batch_size = batch_size
end
def execute
registries = find_registries_pending_verification_on_secondary
return Geo::ProjectRegistry.none if registries.empty?
registries_to_verify = filter_registries_verified_in_primary(registries)
return registries_to_verify unless selective_sync?
legacy_inner_join_registry_ids(
registries_to_verify,
current_node.projects.pluck_primary_key,
Geo::ProjectRegistry,
foreign_key: :project_id
)
end
private
attr_reader :batch_size, :shard_name
# rubocop:disable CodeReuse/ActiveRecord
def find_registries_pending_verification_on_secondary
Geo::ProjectRegistry
.where(Geo::ProjectRegistry.registries_pending_verification)
.pluck(
:project_id,
Geo::ProjectRegistry.repositories_pending_verification.to_sql,
Geo::ProjectRegistry.wikis_pending_verification.to_sql
)
end
# rubocop:enable CodeReuse/ActiveRecord
def filter_registries_verified_in_primary(registries)
filtered_project_ids = filter_projects_verified_on_primary(registries)
Geo::ProjectRegistry.project_id_in(filtered_project_ids)
end
# rubocop:disable CodeReuse/ActiveRecord
def filter_projects_verified_on_primary(registries)
inner_join_project_repository_state(registries)
.joins(:project)
.merge(Project.within_shards(shard_name))
.where(
legacy_repository_state_table[:repository_verification_checksum].not_eq(nil)
.and(project_registry_verify_table[:want_to_verify_repo].eq(true))
.or(legacy_repository_state_table[:wiki_verification_checksum].not_eq(nil)
.and(project_registry_verify_table[:want_to_verify_wiki].eq(true))))
.limit(batch_size)
.pluck_project_key
end
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
def inner_join_project_repository_state(registries)
id_and_want_to_verify = registries.map do |project_id, want_to_verify_repo, want_to_verify_wiki|
"(#{project_id}, #{quote_value(want_to_verify_repo)}, #{quote_value(want_to_verify_wiki)})"
end
ProjectRepositoryState.joins(<<~SQL_REPO)
INNER JOIN
(VALUES #{id_and_want_to_verify.join(',')})
#{project_registry_verify_table.name}(project_id, want_to_verify_repo, want_to_verify_wiki)
ON #{legacy_repository_state_table.name}.project_id = #{project_registry_verify_table.name}.project_id
SQL_REPO
end
# rubocop:enable CodeReuse/ActiveRecord
def legacy_repository_state_table
@legacy_repository_state_table ||= ProjectRepositoryState.arel_table
end
def project_registry_verify_table
@project_registry_verify_table ||= Arel::Table.new(:project_registry_verify_table)
end
end
end
ee/app/finders/geo/legacy_project_registry_retrying_verification_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving project registries that are retrying verification
# scoped to a type (repository or wiki) using cross-database joins
# for selective sync.
#
# Basic usage:
#
# Geo::LegacyProjectRegistryRetryingVerificationFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class LegacyProjectRegistryRetryingVerificationFinder < RegistryFinder
def initialize(current_node: nil, type:)
super(current_node: current_node)
@type = type.to_s.to_sym
end
def execute
legacy_inner_join_registry_ids(
Geo::ProjectRegistry.retrying_verification(type),
current_node.projects.pluck_primary_key,
Geo::ProjectRegistry,
foreign_key: :project_id
)
end
private
attr_reader :type
end
end
ee/app/finders/geo/legacy_project_registry_sync_failed_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving project registries that synchronization have
# failed scoped to a type (repository or wiki) using cross-database
# joins.
#
# Basic usage:
#
# Geo::LegacyProjectRegistrySyncFailedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class LegacyProjectRegistrySyncFailedFinder < RegistryFinder
def initialize(current_node: nil, type:)
super(current_node: current_node)
@type = type.to_s.to_sym
end
def execute
legacy_inner_join_registry_ids(
Geo::ProjectRegistry.sync_failed(type),
current_node.projects.pluck_primary_key,
Geo::ProjectRegistry,
foreign_key: :project_id
)
end
private
attr_reader :type
end
end
ee/app/finders/geo/legacy_project_registry_synced_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving project registries that have been synced
# scoped to a type (repository or wiki) using cross-database joins.
#
# Basic usage:
#
# Geo::LegacyProjectRegistrySyncedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class LegacyProjectRegistrySyncedFinder < RegistryFinder
def initialize(current_node:, type:)
super(current_node: current_node)
@type = type.to_sym
end
def execute
legacy_inner_join_registry_ids(
Geo::ProjectRegistry.synced(type),
current_node.projects.pluck_primary_key,
Geo::ProjectRegistry,
foreign_key: :project_id
)
end
private
attr_reader :type
end
end
ee/app/finders/geo/legacy_project_registry_verification_failed_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving project registries that verification have
# failed scoped to a type (repository or wiki) using cross-database
# joins for selective sync.
#
# Basic usage:
#
# Geo::LegacyProjectRegistryVerificationFailedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class LegacyProjectRegistryVerificationFailedFinder < RegistryFinder
def initialize(current_node: nil, type:)
super(current_node: current_node)
@type = type.to_s.to_sym
end
def execute
legacy_inner_join_registry_ids(
Geo::ProjectRegistry.verification_failed(type),
current_node.projects.pluck_primary_key,
Geo::ProjectRegistry,
foreign_key: :project_id
)
end
private
attr_reader :type
end
end
ee/app/finders/geo/legacy_project_registry_verified_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving project registries that have been verified
# scoped to a type (repository or wiki) using cross-database joins.
#
# Basic usage:
#
# Geo::LegacyProjectRegistryVerifiedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute
#
# Valid `type` values are:
#
# * `:repository`
# * `:wiki`
#
# Any other value will be ignored.
module Geo
class LegacyProjectRegistryVerifiedFinder < RegistryFinder
def initialize(current_node:, type:)
super(current_node: current_node)
@type = type.to_s.to_sym
end
def execute
legacy_inner_join_registry_ids(
Geo::ProjectRegistry.verified(type),
current_node.projects.pluck_primary_key,
Geo::ProjectRegistry,
foreign_key: :project_id
)
end
private
attr_reader :type
end
end
ee/app/finders/geo/legacy_project_unsynced_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving unsynced projects that belong to a specific
# shard using cross-database joins.
#
# Basic usage:
#
# Geo::LegacyProjectUnsyncedFinder
# .new(current_node: Gitlab::Geo.current_node, shard_name: 'default', batch_size: 1000)
# .execute
module Geo
class LegacyProjectUnsyncedFinder < RegistryFinder
def initialize(current_node: nil, shard_name:, batch_size:)
super(current_node: current_node)
@shard_name = shard_name
@batch_size = batch_size
end
# rubocop:disable CodeReuse/ActiveRecord
def execute
legacy_left_outer_join_registry_ids(
current_node.projects.within_shards(shard_name),
Geo::ProjectRegistry.pluck_project_key,
Project
).limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
private
attr_reader :batch_size, :shard_name
end
end
ee/app/finders/geo/legacy_project_updated_recently_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
# Finder for retrieving projects updated recently that
# belong to a specific shard using cross-database joins.
#
# Basic usage:
#
# Geo::LegacyProjectUpdatedRecentlyFinder
# .new(current_node: Gitlab::Geo.current_node, shard_name: 'default', batch_size: 1000)
# .execute
module Geo
class LegacyProjectUpdatedRecentlyFinder < RegistryFinder
def initialize(current_node: nil, shard_name:, batch_size:)
super(current_node: current_node)
@shard_name = shard_name
@batch_size = batch_size
end
# rubocop:disable CodeReuse/ActiveRecord
def execute
registries = find_registries_to_resync
return Project.none if registries.empty?
id_and_last_sync_values = registries.map do |id, last_repository_synced_at|
"(#{id}, #{quote_value(last_repository_synced_at)})"
end
projects = current_node.projects.within_shards(shard_name)
joined_relation = projects.joins(<<~SQL)
INNER JOIN
(VALUES #{id_and_last_sync_values.join(',')})
project_registry(id, last_repository_synced_at)
ON #{Project.table_name}.id = project_registry.id
SQL
joined_relation
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
private
attr_reader :batch_size, :shard_name
# rubocop:disable CodeReuse/ActiveRecord
def find_registries_to_resync
Geo::ProjectRegistry
.dirty
.retry_due
.pluck(:project_id, :last_repository_synced_at)
end
# rubocop:enable CodeReuse/ActiveRecord
end
end
ee/app/finders/geo/project_registry_finder.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
module Geo
class ProjectRegistryFinder < RegistryFinder
def count_projects
current_node.projects.count
end
def count_synced_repositories
registries_for_synced_projects(:repository).count
end
def count_synced_wikis
registries_for_synced_projects(:wiki).count
end
def count_failed_repositories
registries_for_failed_projects(:repository).count
end
def count_failed_wikis
registries_for_failed_projects(:wiki).count
end
def find_failed_project_registries(type = nil)
registries_for_failed_projects(type)
end
def count_verified_repositories
registries_for_verified_projects(:repository).count
end
def count_verified_wikis
registries_for_verified_projects(:wiki).count
end
def count_verification_failed_repositories
registries_for_verification_failed_projects(:repository).count
end
def count_verification_failed_wikis
registries_for_verification_failed_projects(:wiki).count
end
def find_verification_failed_project_registries(type = nil)
registries_for_verification_failed_projects(type)
end
def count_repositories_checksum_mismatch
registries_for_mismatch_projects(:repository).count
end
def count_wikis_checksum_mismatch
registries_for_mismatch_projects(:wiki).count
end
def find_checksum_mismatch_project_registries(type = nil)
registries_for_mismatch_projects(type)
end
def count_repositories_retrying_verification
registries_retrying_verification(:repository).count
end
def count_wikis_retrying_verification
registries_retrying_verification(:wiki).count
end
def find_registries_to_verify(shard_name:, batch_size:)
finder_klass_for_registries_pending_verification
.new(current_node: current_node, shard_name: shard_name, batch_size: batch_size)
.execute
end
def find_unsynced_projects(shard_name:, batch_size:)
finder_klass_for_unsynced_projects
.new(current_node: current_node, shard_name: shard_name, batch_size: batch_size)
.execute
end
def find_projects_updated_recently(shard_name:, batch_size:)
finder_klass_for_projects_updated_recently
.new(current_node: current_node, shard_name: shard_name, batch_size: batch_size)
.execute
end
private
def finder_klass_for_unsynced_projects
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectUnsyncedFinder
else
Geo::ProjectUnsyncedFinder
end
end
def finder_klass_for_projects_updated_recently
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectUpdatedRecentlyFinder
else
Geo::ProjectUpdatedRecentlyFinder
end
end
def finder_klass_for_synced_registries
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistrySyncedFinder
else
Geo::ProjectRegistrySyncedFinder
end
end
def registries_for_synced_projects(type)
finder_klass_for_synced_registries
.new(current_node: current_node, type: type)
.execute
end
def finder_klass_for_failed_registries
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistrySyncFailedFinder
else
Geo::ProjectRegistrySyncFailedFinder
end
end
def registries_for_failed_projects(type)
finder_klass_for_failed_registries
.new(current_node: current_node, type: type)
.execute
end
def finder_klass_for_verified_registries
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistryVerifiedFinder
else
Geo::ProjectRegistryVerifiedFinder
end
end
def registries_for_verified_projects(type)
finder_klass_for_verified_registries
.new(current_node: current_node, type: type)
.execute
end
def finder_klass_for_verification_failed_registries
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistryVerificationFailedFinder
else
Geo::ProjectRegistryVerificationFailedFinder
end
end
def registries_for_verification_failed_projects(type)
finder_klass_for_verification_failed_registries
.new(current_node: current_node, type: type)
.execute
end
def finder_klass_for_registries_retrying_verification
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistryRetryingVerificationFinder
else
Geo::ProjectRegistryRetryingVerificationFinder
end
end
def registries_retrying_verification(type)
finder_klass_for_registries_retrying_verification
.new(current_node: current_node, type: type)
.execute
end
def finder_klass_for_mismatch_registries
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistryMismatchFinder
else
Geo::ProjectRegistryMismatchFinder
end
end
def registries_for_mismatch_projects(type)
finder_klass_for_mismatch_registries
.new(current_node: current_node, type: type)
.execute
end
def finder_klass_for_registries_pending_verification
if use_legacy_queries_for_selective_sync?
Geo::LegacyProjectRegistryPendingVerificationFinder
else
Geo::ProjectRegistryPendingVerificationFinder
end
end
end
end
ee/app/models/geo_node_status.rb
View file @ 7a06ed3f
......@@ -173,6 +173,7 @@ class GeoNodeStatus < ApplicationRecord
self.job_artifact_deleted_max_id = Geo::JobArtifactDeletedEvent.maximum(:id)
self.hashed_storage_migrated_max_id = Geo::HashedStorageMigratedEvent.maximum(:id)
self.hashed_storage_attachments_max_id = Geo::HashedStorageAttachmentsEvent.maximum(:id)
self.projects_count = geo_node.projects.count
load_primary_data
load_secondary_data
......@@ -182,7 +183,6 @@ class GeoNodeStatus < ApplicationRecord
def load_primary_data
if Gitlab::Geo.primary?
self.projects_count = geo_node.projects.count
self.lfs_objects_count = LfsObject.syncable.count
self.job_artifacts_count = Ci::JobArtifact.syncable.count
self.attachments_count = Upload.syncable.count
......@@ -208,11 +208,10 @@ class GeoNodeStatus < ApplicationRecord
self.db_replication_lag_seconds = Gitlab::Geo::HealthCheck.new.db_replication_lag_seconds
self.cursor_last_event_id = current_cursor_last_event_id
self.cursor_last_event_date = Geo::EventLog.find_by(id: self.cursor_last_event_id)&.created_at
self.projects_count = projects_finder.count_projects
self.repositories_synced_count = projects_finder.count_synced_repositories
self.repositories_failed_count = projects_finder.count_failed_repositories
self.wikis_synced_count = projects_finder.count_synced_wikis
self.wikis_failed_count = projects_finder.count_failed_wikis
self.repositories_synced_count = registries_for_synced_projects(:repository).count
self.repositories_failed_count = registries_for_failed_projects(:repository).count
self.wikis_synced_count = registries_for_synced_projects(:wiki).count
self.wikis_failed_count = registries_for_failed_projects(:wiki).count
self.lfs_objects_count = lfs_objects_finder.count_syncable
self.lfs_objects_synced_count = lfs_objects_finder.count_synced
self.lfs_objects_failed_count = lfs_objects_finder.count_failed
......@@ -238,14 +237,14 @@ class GeoNodeStatus < ApplicationRecord
def load_verification_data
if repository_verification_enabled
self.repositories_verified_count = projects_finder.count_verified_repositories
self.repositories_verification_failed_count = projects_finder.count_verification_failed_repositories
self.repositories_checksum_mismatch_count = projects_finder.count_repositories_checksum_mismatch
self.wikis_verified_count = projects_finder.count_verified_wikis
self.wikis_verification_failed_count = projects_finder.count_verification_failed_wikis
self.wikis_checksum_mismatch_count = projects_finder.count_wikis_checksum_mismatch
self.repositories_retrying_verification_count = projects_finder.count_repositories_retrying_verification
self.wikis_retrying_verification_count = projects_finder.count_wikis_retrying_verification
self.repositories_verified_count = registries_for_verified_projects(:repository).count
self.repositories_verification_failed_count = registries_for_verification_failed_projects(:repository).count
self.repositories_checksum_mismatch_count = registries_for_mismatch_projects(:repository).count
self.wikis_verified_count = registries_for_verified_projects(:wiki).count
self.wikis_verification_failed_count = registries_for_verification_failed_projects(:wiki).count
self.wikis_checksum_mismatch_count = registries_for_mismatch_projects(:wiki).count
self.repositories_retrying_verification_count = registries_retrying_verification(:repository).count
self.wikis_retrying_verification_count = registries_retrying_verification(:wiki).count
end
end
......@@ -381,8 +380,40 @@ class GeoNodeStatus < ApplicationRecord
@job_artifacts_finder ||= Geo::JobArtifactRegistryFinder.new(current_node: geo_node)
end
def projects_finder
@projects_finder ||= Geo::ProjectRegistryFinder.new(current_node: geo_node)
def registries_for_synced_projects(type)
Geo::ProjectRegistrySyncedFinder
.new(current_node: geo_node, type: type)
.execute
end
def registries_for_failed_projects(type)
Geo::ProjectRegistrySyncFailedFinder
.new(current_node: geo_node, type: type)
.execute
end
def registries_for_verified_projects(type)
Geo::ProjectRegistryVerifiedFinder
.new(current_node: geo_node, type: type)
.execute
end
def registries_for_verification_failed_projects(type)
Geo::ProjectRegistryVerificationFailedFinder
.new(current_node: geo_node, type: type)
.execute
end
def registries_for_mismatch_projects(type)
Geo::ProjectRegistryMismatchFinder
.new(current_node: geo_node, type: type)
.execute
end
def registries_retrying_verification(type)
Geo::ProjectRegistryRetryingVerificationFinder
.new(current_node: geo_node, type: type)
.execute
end
def repository_verification_finder
......
ee/app/services/geo/files_expire_service.rb
View file @ 7a06ed3f
......@@ -5,6 +5,7 @@ module Geo
include ::Gitlab::Geo::LogHelpers
BATCH_SIZE = 500
UPLOAD_TYPE = 'file'
attr_reader :project, :old_full_path
......@@ -25,7 +26,7 @@ module Geo
def execute
return unless Gitlab::Geo.secondary?
uploads = finder.find_project_uploads(project)
uploads = Geo::Fdw::Upload.for_model_with_type(project, UPLOAD_TYPE)
log_info("Expiring replicated attachments after project rename", count: uploads.count)
schedule_file_removal(uploads)
......@@ -58,11 +59,10 @@ module Geo
# rubocop: enable CodeReuse/ActiveRecord
def mark_for_resync!
finder.find_file_registries_uploads(project).delete_all
end
def finder
@finder ||= ::Geo::ExpireUploadsFinder.new
Gitlab::Geo::Fdw::UploadRegistryQueryBuilder.new
.for_model(project)
.with_type(UPLOAD_TYPE)
.delete_all
end
# This is called by LogHelpers to build json log with context info
......
ee/app/workers/geo/repository_shard_sync_worker.rb
View file @ 7a06ed3f
......@@ -60,10 +60,6 @@ module Geo
scheduled_jobs.map { |data| data[:project_id] }
end
def finder
@finder ||= ProjectRegistryFinder.new(current_node: current_node)
end
def load_pending_resources
resources = find_project_ids_not_synced(batch_size: db_retrieve_batch_size)
remaining_capacity = db_retrieve_batch_size - resources.size
......@@ -77,20 +73,32 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_not_synced(batch_size:)
finder.find_unsynced_projects(shard_name: shard_name, batch_size: batch_size)
find_unsynced_projects(batch_size: batch_size)
.id_not_in(scheduled_project_ids)
.reorder(last_repository_updated_at: :desc)
.pluck_primary_key
end
# rubocop: enable CodeReuse/ActiveRecord
def find_unsynced_projects(batch_size:)
Geo::ProjectUnsyncedFinder
.new(current_node: current_node, shard_name: shard_name, batch_size: batch_size)
.execute
end
# rubocop: disable CodeReuse/ActiveRecord
def find_project_ids_updated_recently(batch_size:)
finder.find_projects_updated_recently(shard_name: shard_name, batch_size: batch_size)
find_projects_updated_recently(batch_size: batch_size)
.id_not_in(scheduled_project_ids)
.order('project_registry.last_repository_synced_at ASC NULLS FIRST, projects.last_repository_updated_at ASC')
.pluck_primary_key
end
# rubocop: enable CodeReuse/ActiveRecord
def find_projects_updated_recently(batch_size:)
Geo::ProjectUpdatedRecentlyFinder
.new(current_node: current_node, shard_name: shard_name, batch_size: batch_size)
.execute
end
end
end
ee/app/workers/geo/repository_verification/secondary/shard_worker.rb
View file @ 7a06ed3f
......@@ -33,22 +33,18 @@ module Geo
current_node.verification_max_capacity
end
# rubocop: disable CodeReuse/ActiveRecord
def load_pending_resources
finder.find_registries_to_verify(shard_name: shard_name, batch_size: db_retrieve_batch_size)
.pluck(:id)
Geo::ProjectRegistryPendingVerificationFinder
.new(current_node: current_node, shard_name: shard_name, batch_size: db_retrieve_batch_size)
.execute
.pluck_primary_key
end
# rubocop: enable CodeReuse/ActiveRecord
def schedule_job(registry_id)
job_id = Geo::RepositoryVerification::Secondary::SingleWorker.perform_async(registry_id)
{ id: registry_id, job_id: job_id } if job_id
end
def finder
@finder ||= Geo::ProjectRegistryFinder.new(current_node: current_node)
end
end
end
end
......
ee/lib/api/geo_nodes.rb
View file @ 7a06ed3f
......@@ -49,24 +49,21 @@ module API
use :pagination
end
get '/current/failures' do
geo_node = Gitlab::Geo.current_node
forbidden!('Failures can only be requested from a secondary node') unless geo_node.secondary?
not_found!('Geo node not found') unless geo_node
finder = ::Geo::ProjectRegistryFinder.new(current_node: geo_node)
project_registries = case params[:failure_type]
when 'sync'
finder.find_failed_project_registries(params[:type])
when 'verification'
finder.find_verification_failed_project_registries(params[:type])
when 'checksum_mismatch'
finder.find_checksum_mismatch_project_registries(params[:type])
else
not_found!('Failure type unknown')
end
not_found!('Geo node not found') unless Gitlab::Geo.current_node
forbidden!('Failures can only be requested from a secondary node') unless Gitlab::Geo.current_node.secondary?
finder_klass = case params[:failure_type]
when 'sync'
::Geo::ProjectRegistrySyncFailedFinder
when 'verification'
::Geo::ProjectRegistryVerificationFailedFinder
when 'checksum_mismatch'
::Geo::ProjectRegistryMismatchFinder
else
not_found!('Failure type unknown')
end
project_registries = finder_klass.new(current_node: Gitlab::Geo.current_node, type: params[:type]).execute
present paginate(project_registries), with: ::GeoProjectRegistryEntity
end
......
ee/spec/finders/geo/attachment_registry_finder_spec.rb
View file @ 7a06ed3f
require 'spec_helper'
describe Geo::AttachmentRegistryFinder, :geo do
describe Geo::AttachmentRegistryFinder, :geo, :geo_fdw do
include ::EE::GeoHelpers
# Using let() instead of set() because set() does not work properly
# when using the :delete DatabaseCleaner strategy, which is required for FDW
# tests because a foreign table can't see changes inside a transaction of a
# different connection.
# when using the :delete DatabaseCleaner strategy, which is required
# for FDW tests because a foreign table can't see changes inside a
# transaction of a different connection.
let(:secondary) { create(:geo_node) }
let(:synced_group) { create(:group) }
......@@ -129,56 +129,6 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
shared_examples 'counts all the things' do
describe '#count_syncable' do
let!(:upload_1) { create(:upload, model: synced_group) }
let!(:upload_2) { create(:upload, model: unsynced_group) }
let!(:upload_3) { create(:upload, :issuable_upload, model: synced_project_in_nested_group) }
let!(:upload_4) { create(:upload, model: unsynced_project) }
let!(:upload_5) { create(:upload, :personal_snippet_upload) }
it 'counts attachments' do
expect(subject.count_syncable).to eq 5
end
it 'ignores remote attachments' do
upload_1.update!(store: ObjectStorage::Store::REMOTE)
expect(subject.count_syncable).to eq 4
end
context 'with selective sync by namespace' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'counts attachments' do
expect(subject.count_syncable).to eq 3
end
it 'ignores remote attachments' do
upload_1.update!(store: ObjectStorage::Store::REMOTE)
expect(subject.count_syncable).to eq 2
end
end
context 'with selective sync by shard' do
before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'counts attachments' do
expect(subject.count_syncable).to eq 3
end
it 'ignores remote attachments' do
upload_4.update!(store: ObjectStorage::Store::REMOTE)
expect(subject.count_syncable).to eq 2
end
end
end
describe '#count_synced' do
let!(:upload_1) { create(:upload, model: synced_group) }
let!(:upload_2) { create(:upload, model: unsynced_group) }
......@@ -404,6 +354,56 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
end
describe '#count_syncable' do
let!(:upload_1) { create(:upload, model: synced_group) }
let!(:upload_2) { create(:upload, model: unsynced_group) }
let!(:upload_3) { create(:upload, :issuable_upload, model: synced_project_in_nested_group) }
let!(:upload_4) { create(:upload, model: unsynced_project) }
let!(:upload_5) { create(:upload, :personal_snippet_upload) }
it 'counts attachments' do
expect(subject.count_syncable).to eq 5
end
it 'ignores remote attachments' do
upload_1.update!(store: ObjectStorage::Store::REMOTE)
expect(subject.count_syncable).to eq 4
end
context 'with selective sync by namespace' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'counts attachments' do
expect(subject.count_syncable).to eq 3
end
it 'ignores remote attachments' do
upload_1.update!(store: ObjectStorage::Store::REMOTE)
expect(subject.count_syncable).to eq 2
end
end
context 'with selective sync by shard' do
before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'counts attachments' do
expect(subject.count_syncable).to eq 3
end
it 'ignores remote attachments' do
upload_4.update!(store: ObjectStorage::Store::REMOTE)
expect(subject.count_syncable).to eq 2
end
end
end
describe '#count_registry' do
let!(:upload_1) { create(:upload, model: synced_group) }
let!(:upload_2) { create(:upload, model: unsynced_group) }
......@@ -456,5 +456,25 @@ describe Geo::AttachmentRegistryFinder, :geo do
end
end
it_behaves_like 'a file registry finder'
it 'responds to file registry finder methods' do
file_registry_finder_methods = %i{
syncable
count_syncable
count_synced
count_failed
count_synced_missing_on_primary
count_registry
find_unsynced
find_migrated_local
find_retryable_failed_registries
find_retryable_synced_missing_on_primary_registries
}
file_registry_finder_methods.each do |method|
expect(subject).to respond_to(method)
end
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
ee/spec/finders/geo/expire_uploads_finder_spec.rb deleted 100644 → 0
View file @ 47592413
require 'spec_helper'
describe Geo::ExpireUploadsFinder, :geo do
include EE::GeoHelpers
let(:project) { create(:project) }
context 'FDW', :geo_fdw do
describe '#find_project_uploads' do
context 'filtering per project uploads' do
it 'returns only objects associated with the project' do
other_upload = create(:upload, :issuable_upload)
upload = create(:upload, :issuable_upload, model: project)
create(:geo_file_registry, file_id: upload.id)
create(:geo_file_registry, file_id: other_upload.id)
uploads = subject.find_project_uploads(project)
expect(uploads.count).to eq(1)
expect(uploads.first.id).to eq(upload.id)
end
end
context 'filtering replicated uploads only' do
it 'returns only replicated or to be replicated objects' do
create(:upload, :issuable_upload, model: project)
upload = create(:upload, :issuable_upload, model: project)
create(:geo_file_registry, file_id: upload.id, success: false)
uploads = subject.find_project_uploads(project)
expect(uploads.count).to eq(1)
expect(uploads.first.id).to eq(upload.id)
end
end
end
describe '#find_file_registries_uploads' do
context 'filtering per project uploads' do
it 'returns only objects associated with the project' do
other_upload = create(:upload, :issuable_upload)
upload = create(:upload, :issuable_upload, model: project)
create(:geo_file_registry, file_id: other_upload.id)
file_registry = create(:geo_file_registry, file_id: upload.id)
files = subject.find_file_registries_uploads(project)
expect(files.count).to eq(1)
expect(files.first.id).to eq(file_registry.id)
end
end
end
end
context 'Legacy' do
before do
stub_fdw_disabled
end
describe '#find_project_uploads' do
context 'filtering per project uploads' do
it 'returns only objects associated with the project' do
other_upload = create(:upload, :issuable_upload)
upload = create(:upload, :issuable_upload, model: project)
create(:geo_file_registry, file_id: upload.id)
create(:geo_file_registry, file_id: other_upload.id)
uploads = subject.find_project_uploads(project)
expect(uploads.count).to eq(1)
expect(uploads.first.id).to eq(upload.id)
end
end
context 'filtering replicated uploads only' do
it 'returns only replicated or to be replicated objects' do
create(:upload, :issuable_upload, model: project)
upload = create(:upload, :issuable_upload, model: project)
create(:geo_file_registry, file_id: upload.id, success: false)
uploads = subject.find_project_uploads(project)
expect(uploads.count).to eq(1)
expect(uploads.first.id).to eq(upload.id)
end
end
end
describe '#find_file_registries_uploads' do
context 'filtering per project uploads' do
it 'returns only objects associated with the project' do
other_upload = create(:upload, :issuable_upload)
upload = create(:upload, :issuable_upload, model: project)
create(:geo_file_registry, file_id: other_upload.id)
file_registry = create(:geo_file_registry, file_id: upload.id)
files = subject.find_file_registries_uploads(project)
expect(files.count).to eq(1)
expect(files.first.id).to eq(file_registry.id)
end
end
end
end
end
ee/spec/finders/geo/legacy_project_registry_mismatch_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectRegistryMismatchFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_mismatch) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_checksum_mismatch, project: project_1) }
let!(:registry_repository_mismatch) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_verified, project: project_2) }
let!(:registry_wiki_mismatch) { create(:geo_project_registry, :repository_verified, :wiki_checksum_mismatch, project: project_3) }
let!(:registry_wiki_mismatch_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_checksum_mismatch, project: project_4) }
let!(:registry_repository_mismatch_broken_shard) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_verified, project: project_5) }
let!(:registry_verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all mismatch registries' do
expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_repository_mismatch_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns mismatch registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch)
end
end
context 'with selective sync by shard' do
it 'returns mismatch registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_mismatch_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all mismatch registries' do
expect(subject.execute).to contain_exactly(registry_mismatch, registry_wiki_mismatch, registry_wiki_mismatch_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns mismatch registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_mismatch, registry_wiki_mismatch)
end
end
context 'with selective sync by shard' do
it 'returns mismatch registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_wiki_mismatch_broken_shard)
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns all mismatch registries' do
expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_wiki_mismatch, registry_repository_mismatch_broken_shard, registry_wiki_mismatch_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns all mismatch registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_wiki_mismatch)
end
end
context 'with selective sync by shard' do
it 'returns all mismatch registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_mismatch_broken_shard, registry_wiki_mismatch_broken_shard)
end
end
end
end
end
ee/spec/finders/geo/legacy_project_registry_pending_verification_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectRegistryPendingVerificationFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
subject { described_class.new(current_node: node, shard_name: 'default', batch_size: 100) }
it 'does not return registries that are verified on primary and secondary' do
project_verified = create(:repository_state, :repository_verified, :wiki_verified).project
repository_verified = create(:repository_state, :repository_verified).project
wiki_verified = create(:repository_state, :wiki_verified).project
create(:geo_project_registry, :repository_verified, :wiki_verified, project: project_verified)
create(:geo_project_registry, :repository_verified, project: repository_verified)
create(:geo_project_registry, :wiki_verified, project: wiki_verified)
expect(subject.execute).to be_empty
end
it 'does not return registries that were unverified/outdated on primary' do
project_unverified_primary = create(:project)
project_outdated_primary = create(:repository_state, :repository_outdated, :wiki_outdated).project
repository_outdated_primary = create(:repository_state, :repository_outdated, :wiki_verified).project
wiki_outdated_primary = create(:repository_state, :repository_verified, :wiki_outdated).project
create(:geo_project_registry, project: project_unverified_primary)
create(:geo_project_registry, :repository_verification_outdated, :wiki_verification_outdated, project: project_outdated_primary)
create(:geo_project_registry, :repository_verified, :wiki_verified, project: repository_outdated_primary)
create(:geo_project_registry, :repository_verified, :wiki_verified, project: wiki_outdated_primary)
expect(subject.execute).to be_empty
end
it 'returns registries that were unverified/outdated on secondary' do
project_unverified_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
project_outdated_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
repository_outdated_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
wiki_outdated_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
registry_unverified_secondary = create(:geo_project_registry, :synced, project: project_unverified_secondary)
registry_outdated_secondary = create(:geo_project_registry, :synced, :repository_verification_outdated, :wiki_verification_outdated, project: project_outdated_secondary)
registry_repository_outdated_secondary = create(:geo_project_registry, :synced, :repository_verification_outdated, :wiki_verified, project: repository_outdated_secondary)
registry_wiki_outdated_secondary = create(:geo_project_registry, :synced, :repository_verified, :wiki_verification_outdated, project: wiki_outdated_secondary)
expect(subject.execute)
.to contain_exactly(
registry_unverified_secondary,
registry_outdated_secondary,
registry_repository_outdated_secondary,
registry_wiki_outdated_secondary
)
end
it 'does not return registries that failed on primary' do
verification_failed_primary = create(:repository_state, :repository_failed, :wiki_failed).project
create(:geo_project_registry, project: verification_failed_primary)
expect(subject.execute).to be_empty
end
it 'returns registries where one failed and one verified on the primary' do
verification_failed_primary = create(:repository_state, :repository_failed, :wiki_failed).project
repository_failed_primary = create(:repository_state, :repository_failed, :wiki_verified).project
wiki_failed_primary = create(:repository_state, :repository_verified, :wiki_failed).project
create(:geo_project_registry, :synced, project: verification_failed_primary)
registry_repository_failed_primary = create(:geo_project_registry, :synced, project: repository_failed_primary)
registry_wiki_failed_primary = create(:geo_project_registry, :synced, project: wiki_failed_primary)
expect(subject.execute)
.to contain_exactly(
registry_repository_failed_primary,
registry_wiki_failed_primary
)
end
it 'does not return registries where verification failed on secondary' do
verification_failed_secondary = create(:repository_state, :repository_verified, :wiki_verified).project
repository_failed_secondary = create(:repository_state, :repository_verified).project
wiki_failed_secondary = create(:repository_state, :wiki_verified).project
create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed, project: verification_failed_secondary)
create(:geo_project_registry, :repository_verification_failed, project: repository_failed_secondary)
create(:geo_project_registry, :wiki_verification_failed, project: wiki_failed_secondary)
expect(subject.execute).to be_empty
end
it 'does not return registries when the repo needs to be resynced' do
project_verified = create(:repository_state, :repository_verified).project
create(:geo_project_registry, :repository_sync_failed, project: project_verified)
expect(subject.execute).to be_empty
end
it 'does not return registries when the wiki needs to be resynced' do
project_verified = create(:repository_state, :wiki_verified).project
create(:geo_project_registry, :wiki_sync_failed, project: project_verified)
expect(subject.execute).to be_empty
end
it 'does not return registries when the repository is missing on primary' do
project_verified = create(:repository_state, :repository_verified).project
create(:geo_project_registry, :synced, project: project_verified, repository_missing_on_primary: true)
expect(subject.execute).to be_empty
end
it 'does not return registries when the wiki is missing on primary' do
project_verified = create(:repository_state, :wiki_verified).project
create(:geo_project_registry, :synced, project: project_verified, wiki_missing_on_primary: true)
expect(subject.execute).to be_empty
end
it 'does not return registries where projects belongs to other shards' do
project_broken_storage = create(:project, :broken_storage)
create(:repository_state, :repository_verified, :wiki_verified, project: project_broken_storage)
create(:geo_project_registry, :synced, project: project_broken_storage)
expect(subject.execute).to be_empty
end
context 'with selective sync by namespace' do
it 'returns registries where projects belongs to the namespaces' do
group_1 = create(:group)
group_2 = create(:group)
nested_group_1 = create(:group, parent: group_1)
project_1 = create(:project, group: group_1)
project_2 = create(:project, group: nested_group_1)
project_3 = create(:project, group: group_2)
create(:repository_state, :repository_verified, :wiki_verified, project: project_1)
create(:repository_state, :repository_verified, :wiki_verified, project: project_2)
create(:repository_state, :repository_verified, :wiki_verified, project: project_3)
registry_unverified_secondary = create(:geo_project_registry, :synced, project: project_1)
registry_outdated_secondary = create(:geo_project_registry, :synced, :repository_verification_outdated, :wiki_verification_outdated, project: project_2)
create(:geo_project_registry, :synced, :repository_verification_outdated, :wiki_verified, project: project_3)
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute)
.to contain_exactly(
registry_unverified_secondary,
registry_outdated_secondary
)
end
end
context 'with selective sync by shard' do
let(:project_broken_storage) { create(:project, :broken_storage) }
let!(:repository_state_project_broken_storage) { create(:repository_state, :repository_verified, :wiki_verified, project: project_broken_storage) }
let!(:registry_repository_broken_shard) { create(:geo_project_registry, :synced, project: project_broken_storage) }
let(:project) { create(:project) }
let!(:project_unverified_secondary) { create(:repository_state, :repository_verified, :wiki_verified, project: project) }
let!(:registry_unverified_secondary) { create(:geo_project_registry, :synced, project: project) }
before do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'does not return registries when selected shards to sync does not include the shard_name' do
subject = described_class.new(current_node: node, shard_name: 'default', batch_size: 100)
expect(subject.execute).to be_empty
end
it 'returns registries where projects belongs to the shards' do
subject = described_class.new(current_node: node, shard_name: 'broken', batch_size: 100)
expect(subject.execute).to contain_exactly(registry_repository_broken_shard)
end
end
end
end
ee/spec/finders/geo/legacy_project_registry_retrying_verification_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectRegistryRetryingVerificationFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:retrying_verification) { create(:geo_project_registry, :repository_retrying_verification, :wiki_retrying_verification, project: project_1) }
let!(:repository_retrying_verification) { create(:geo_project_registry, :repository_retrying_verification, :wiki_verified, project: project_2) }
let!(:wiki_retrying_verification) { create(:geo_project_registry, :repository_verified, :wiki_retrying_verification, project: project_3) }
let!(:wiki_retrying_verification_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_retrying_verification, project: project_4) }
let!(:repository_retrying_verification_broken_shard) { create(:geo_project_registry, :repository_retrying_verification, :wiki_verified, project: project_5) }
let!(:verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all registries retrying verification' do
expect(subject.execute).to contain_exactly(retrying_verification, repository_retrying_verification, repository_retrying_verification_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns registries retrying verification where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(retrying_verification, repository_retrying_verification)
end
end
context 'with selective sync by shard' do
it 'returns registries retrying verification where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(repository_retrying_verification_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all registries retrying verification' do
expect(subject.execute).to contain_exactly(retrying_verification, wiki_retrying_verification, wiki_retrying_verification_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns registries retrying verification where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(retrying_verification, wiki_retrying_verification)
end
end
context 'with selective sync by shard' do
it 'returns registries retrying verification where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(wiki_retrying_verification_broken_shard)
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
context 'with selective sync by namespace' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
context 'with selective sync by shard' do
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
end
end
end
ee/spec/finders/geo/legacy_project_registry_sync_failed_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectRegistrySyncFailedFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_failed) { create(:geo_project_registry, :sync_failed, project: project_1) }
let!(:registry_repository_failed) { create(:geo_project_registry, :synced, :repository_sync_failed, project: project_2) }
let!(:registry_wiki_failed) { create(:geo_project_registry, :synced, :wiki_sync_failed, project: project_3) }
let!(:registry_wiki_failed_broken_shard) { create(:geo_project_registry, :synced, :wiki_sync_failed, project: project_4) }
let!(:registry_repository_failed_broken_shard) { create(:geo_project_registry, :synced, :repository_sync_failed, project: project_5) }
let!(:registry_synced) { create(:geo_project_registry, :synced) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_repository_failed_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_failed, registry_repository_failed])
end
end
context 'with selective sync by shard' do
it 'returns failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_repository_failed_broken_shard])
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to match_array([registry_failed, registry_wiki_failed, registry_wiki_failed_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_failed, registry_wiki_failed])
end
end
context 'with selective sync by shard' do
it 'returns failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_wiki_failed_broken_shard])
end
end
end
context 'with no type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_wiki_failed, registry_repository_failed_broken_shard, registry_wiki_failed_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns all failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_wiki_failed])
end
end
context 'with selective sync by shard' do
it 'returns all failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_repository_failed_broken_shard, registry_wiki_failed_broken_shard])
end
end
end
end
end
ee/spec/finders/geo/legacy_project_registry_synced_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectRegistrySyncedFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_synced) { create(:geo_project_registry, :synced, project: project_1) }
let!(:registry_repository_dirty) { create(:geo_project_registry, :synced, :repository_dirty, project: project_2) }
let!(:registry_wiki_dirty) { create(:geo_project_registry, :synced, :wiki_dirty, project: project_3) }
let!(:registry_wiki_dirty_broken_shard) { create(:geo_project_registry, :synced, :wiki_dirty, project: project_4) }
let!(:registry_repository_dirty_broken_shard) { create(:geo_project_registry, :synced, :repository_dirty, project: project_5) }
let!(:registry_sync_failed) { create(:geo_project_registry, :sync_failed) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all synced registries' do
expect(subject.execute).to match_array([registry_synced, registry_wiki_dirty, registry_wiki_dirty_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns synced registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_synced, registry_wiki_dirty])
end
end
context 'with selective sync by shard' do
it 'returns synced registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_wiki_dirty_broken_shard])
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all synced registries' do
expect(subject.execute).to match_array([registry_synced, registry_repository_dirty, registry_repository_dirty_broken_shard])
end
end
context 'with selective sync by namespace' do
it 'returns synced registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_array([registry_synced, registry_repository_dirty])
end
end
context 'with selective sync by shard' do
it 'returns synced registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to match_array([registry_repository_dirty_broken_shard])
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
end
end
ee/spec/finders/geo/legacy_project_registry_verification_failed_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectRegistryVerificationFailedFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed, project: project_1) }
let!(:registry_repository_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_2) }
let!(:registry_wiki_verification_failed) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_3) }
let!(:registry_wiki_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_4) }
let!(:registry_repository_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_5) }
let!(:registry_verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_repository_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_verification_failed_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_wiki_verification_failed, registry_wiki_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_wiki_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_wiki_verification_failed_broken_shard)
end
end
end
context 'with no type' do
subject { described_class.new(current_node: node, type: :invalid) }
context 'without selective sync' do
it 'returns all failed registries' do
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_wiki_verification_failed, registry_repository_verification_failed_broken_shard, registry_wiki_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns all failed registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_wiki_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns all failed registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_verification_failed_broken_shard, registry_wiki_verification_failed_broken_shard)
end
end
end
end
end
ee/spec/finders/geo/legacy_project_registry_verified_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectRegistryVerifiedFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let(:project_1) { create(:project, group: group_1) }
let(:project_2) { create(:project, group: nested_group_1) }
let(:project_3) { create(:project, group: nested_group_1) }
let(:project_4) { create(:project, :broken_storage, group: group_2) }
let(:project_5) { create(:project, :broken_storage, group: group_2) }
let!(:registry_verified) { create(:geo_project_registry, :repository_verified, :wiki_verified, project: project_1) }
let!(:registry_repository_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_2) }
let!(:registry_repository_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_5) }
let!(:registry_wiki_verification_failed) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_3) }
let!(:registry_wiki_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_4) }
let!(:registry_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed) }
context 'with repository type' do
subject { described_class.new(current_node: node, type: :repository) }
context 'without selective sync' do
it 'returns all verified registries' do
expect(subject.execute).to contain_exactly(registry_verified, registry_wiki_verification_failed, registry_wiki_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns verified registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verified, registry_wiki_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns verified registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_wiki_verification_failed_broken_shard)
end
end
end
context 'with wiki type' do
subject { described_class.new(current_node: node, type: :wiki) }
context 'without selective sync' do
it 'returns all verified registries' do
expect(subject.execute).to contain_exactly(registry_verified, registry_repository_verification_failed, registry_repository_verification_failed_broken_shard)
end
end
context 'with selective sync by namespace' do
it 'returns verified registries where projects belongs to the namespaces' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to contain_exactly(registry_verified, registry_repository_verification_failed)
end
end
context 'with selective sync by shard' do
it 'returns verified registries where projects belongs to the shards' do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.execute).to contain_exactly(registry_repository_verification_failed_broken_shard)
end
end
end
context 'with invalid type' do
subject { described_class.new(current_node: node, type: :invalid) }
it 'returns nothing' do
expect(subject.execute).to be_empty
end
end
end
end
ee/spec/finders/geo/legacy_project_unsynced_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectUnsyncedFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let!(:project_1) { create(:project, group: group_1) }
let!(:project_2) { create(:project, group: nested_group_1) }
let!(:project_3) { create(:project, group: group_2) }
let!(:project_4) { create(:project, group: group_1) }
before do
project_4.update_column(:repository_storage, 'foo')
end
subject { described_class.new(current_node: node, shard_name: 'default', batch_size: 100) }
context 'without selective sync' do
it 'returns projects without an entry on the tracking database' do
create(:geo_project_registry, :synced, project: project_2)
expect(subject.execute).to match_ids(project_1, project_3)
end
end
context 'with selective sync by namespace' do
it 'returns projects that belong to the namespaces without an entry on the tracking database' do
create(:geo_project_registry, :synced, project: project_4)
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1, nested_group_1])
expect(subject.execute).to match_ids(project_1, project_2)
end
end
context 'with selective sync by shard' do
before do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['foo'])
end
it 'does not return registries when selected shards to sync does not include the shard_name' do
subject = described_class.new(current_node: node, shard_name: 'default', batch_size: 100)
expect(subject.execute).to be_empty
end
it 'returns projects that belong to the shards without an entry on the tracking database' do
project_5 = create(:project, group: group_1)
project_5.update_column(:repository_storage, 'foo')
create(:geo_project_registry, :synced, project: project_4)
subject = described_class.new(current_node: node, shard_name: 'foo', batch_size: 100)
expect(subject.execute).to match_ids(project_5)
end
end
end
end
ee/spec/finders/geo/legacy_project_updated_recently_finder_spec.rb deleted 100644 → 0
View file @ 47592413
# frozen_string_literal: true
require 'spec_helper'
describe Geo::LegacyProjectUpdatedRecentlyFinder, :geo do
describe '#execute' do
let(:node) { create(:geo_node) }
let(:group_1) { create(:group) }
let(:group_2) { create(:group) }
let(:nested_group_1) { create(:group, parent: group_1) }
let!(:project_1) { create(:project, group: group_1) }
let!(:project_2) { create(:project, group: nested_group_1) }
let!(:project_3) { create(:project, group: group_2) }
let!(:project_4) { create(:project, group: group_1) }
before do
project_4.update_column(:repository_storage, 'foo')
create(:geo_project_registry, :synced, :repository_dirty, project: project_1)
create(:geo_project_registry, :synced, :repository_dirty, project: project_2)
create(:geo_project_registry, :synced, project: project_3)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_4)
end
subject { described_class.new(current_node: node, shard_name: 'default', batch_size: 100) }
context 'without selective sync' do
it 'returns projects with a dirty entry on the tracking database' do
expect(subject.execute).to match_ids(project_1, project_2)
end
end
context 'with selective sync by namespace' do
it 'returns projects that belong to the namespaces with a dirty entry on the tracking database' do
node.update!(selective_sync_type: 'namespaces', namespaces: [group_1])
expect(subject.execute).to match_ids(project_1, project_2)
end
end
context 'with selective sync by shard' do
before do
node.update!(selective_sync_type: 'shards', selective_sync_shards: ['foo'])
end
it 'does not return registries when selected shards to sync does not include the shard_name' do
subject = described_class.new(current_node: node, shard_name: 'default', batch_size: 100)
expect(subject.execute).to be_empty
end
it 'returns projects that belong to the shards with a dirty entry on the tracking database' do
project_5 = create(:project, group: group_1)
project_5.update_column(:repository_storage, 'foo')
create(:geo_project_registry, :synced, project: project_5)
subject = described_class.new(current_node: node, shard_name: 'foo', batch_size: 100)
expect(subject.execute).to match_ids(project_4)
end
end
end
end
ee/spec/finders/geo/project_registry_finder_spec.rb deleted 100644 → 0
View file @ 47592413
require 'spec_helper'
describe Geo::ProjectRegistryFinder, :geo do
using RSpec::Parameterized::TableSyntax
include ::EE::GeoHelpers
# Using let() instead of set() because set() does not work properly
# when using the :delete DatabaseCleaner strategy, which is required for FDW
# tests because a foreign table can't see changes inside a transaction of a
# different connection.
let(:secondary) { create(:geo_node) }
let(:synced_group) { create(:group) }
let(:nested_group_1) { create(:group, parent: synced_group) }
let(:project) { create(:project) }
let(:project_1_in_synced_group) { create(:project, group: synced_group) }
let(:project_2_in_synced_group) { create(:project, group: nested_group_1) }
let(:project_3_in_synced_group) { create(:project, group: synced_group) }
let(:project_4_broken_storage) { create(:project, :broken_storage) }
subject { described_class.new(current_node: secondary) }
before do
stub_current_geo_node(secondary)
end
shared_examples 'counts all the things' do |method_prefix|
describe '#count_synced_repositories' do
before do
create(:geo_project_registry, :synced, project: project)
create(:geo_project_registry, :synced, :repository_dirty, project: project_1_in_synced_group)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_2_in_synced_group)
create(:geo_project_registry, :sync_failed, project: project_3_in_synced_group)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_4_broken_storage)
end
it 'counts registries that repository have been synced' do
expect(subject.count_synced_repositories).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that repository have been synced where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_synced_repositories).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts registries that repository have been synced where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_synced_repositories).to eq 1
end
end
end
describe '#count_synced_wikis' do
before do
create(:geo_project_registry, :synced, project: project)
create(:geo_project_registry, :synced, :wiki_dirty, project: project_1_in_synced_group)
create(:geo_project_registry, :synced, :repository_dirty, project: project_2_in_synced_group)
create(:geo_project_registry, :sync_failed, project: project_3_in_synced_group)
create(:geo_project_registry, :synced, :repository_dirty, project: project_4_broken_storage)
end
it 'counts registries that wiki have been synced' do
expect(subject.count_synced_wikis).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that wiki have been synced where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_synced_wikis).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts registries that wiki have been synced where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_synced_wikis).to eq 1
end
end
end
describe '#count_failed_repositories' do
before do
create(:geo_project_registry, :synced, project: project)
create(:geo_project_registry, :repository_sync_failed, project: project_1_in_synced_group)
create(:geo_project_registry, :wiki_sync_failed, project: project_2_in_synced_group)
create(:geo_project_registry, :sync_failed, project: project_3_in_synced_group)
create(:geo_project_registry, :repository_sync_failed, project: project_4_broken_storage)
end
it 'counts registries that repository sync has failed' do
expect(subject.count_failed_repositories).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that repository sync has failed where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_failed_repositories).to eq 2
end
end
context 'with selective sync by shard' do
it 'counts registries that repository sync has failed where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_failed_repositories).to eq 1
end
end
end
describe '#count_failed_wikis' do
before do
create(:geo_project_registry, :synced, project: project)
create(:geo_project_registry, :wiki_sync_failed, project: project_1_in_synced_group)
create(:geo_project_registry, :repository_sync_failed, project: project_2_in_synced_group)
create(:geo_project_registry, :sync_failed, project: project_3_in_synced_group)
create(:geo_project_registry, :wiki_sync_failed, project: project_4_broken_storage)
end
it 'counts registries that wiki sync has failed' do
expect(subject.count_failed_wikis).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that wiki sync has failed where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_failed_wikis).to eq 2
end
end
context 'with selective sync by shard' do
it 'counts registries that wiki sync has failed where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_failed_wikis).to eq 1
end
end
end
describe '#count_verified_repositories' do
before do
create(:geo_project_registry, :repository_verified, :wiki_verified, project: project)
create(:geo_project_registry, :repository_verified, project: project_1_in_synced_group)
create(:geo_project_registry, :repository_verification_failed, project: project_3_in_synced_group)
create(:geo_project_registry, :repository_verified, project: project_4_broken_storage)
create(:geo_project_registry, :wiki_verified, project: project_2_in_synced_group)
end
it 'counts registries that repository have beend verified' do
expect(subject.count_verified_repositories).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that repository have beend verified where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_verified_repositories).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts registries that repository have beend verified where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_verified_repositories).to eq 1
end
end
end
describe '#count_verified_wikis' do
before do
create(:geo_project_registry, :wiki_verified, :wiki_verified, project: project)
create(:geo_project_registry, :wiki_verified, project: project_1_in_synced_group)
create(:geo_project_registry, :wiki_verification_failed, project: project_3_in_synced_group)
create(:geo_project_registry, :wiki_verified, project: project_4_broken_storage)
create(:geo_project_registry, :repository_verified, project: project_2_in_synced_group)
end
it 'counts registries that wiki have beend verified' do
expect(subject.count_verified_wikis).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that wiki have beend verified where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_verified_wikis).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts registries that wiki have beend verified where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_verified_wikis).to eq 1
end
end
end
describe '#count_verification_failed_repositories' do
before do
create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed, project: project)
create(:geo_project_registry, :repository_verification_failed, project: project_1_in_synced_group)
create(:geo_project_registry, :repository_verification_failed, project: project_4_broken_storage)
create(:geo_project_registry, :wiki_verification_failed, project: project_2_in_synced_group)
end
it 'counts registries that repository verification has failed' do
expect(subject.count_verification_failed_repositories).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that repository verification has failed where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_verification_failed_repositories).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts registries that repository verification has failed where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_verification_failed_repositories).to eq 1
end
end
end
describe '#count_verification_failed_wikis' do
before do
create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed, project: project)
create(:geo_project_registry, :wiki_verification_failed, project: project_1_in_synced_group)
create(:geo_project_registry, :wiki_verification_failed, project: project_4_broken_storage)
create(:geo_project_registry, :repository_verification_failed, project: project_2_in_synced_group)
end
it 'counts registries that wiki verification has failed' do
expect(subject.count_verification_failed_wikis).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that wiki verification has failed where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_verification_failed_wikis).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts registries that wiki verification has failed where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_verification_failed_wikis).to eq 1
end
end
end
describe '#count_repositories_retrying_verification' do
before do
create(:geo_project_registry, :repository_retrying_verification, :wiki_retrying_verification, project: project)
create(:geo_project_registry, :repository_retrying_verification, project: project_1_in_synced_group)
create(:geo_project_registry, :repository_retrying_verification, project: project_4_broken_storage)
create(:geo_project_registry, :wiki_retrying_verification, project: project_2_in_synced_group)
end
it 'counts registries that repository retrying verification' do
expect(subject.count_repositories_retrying_verification).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that repository retrying verification where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_repositories_retrying_verification).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts registries that repository retrying verification where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_repositories_retrying_verification).to eq 1
end
end
end
describe '#count_wikis_retrying_verification' do
before do
create(:geo_project_registry, :repository_retrying_verification, :wiki_retrying_verification, project: project)
create(:geo_project_registry, :repository_retrying_verification, project: project_1_in_synced_group)
create(:geo_project_registry, :wiki_retrying_verification, project: project_2_in_synced_group)
create(:geo_project_registry, :wiki_retrying_verification, project: project_4_broken_storage)
end
it 'counts registries that wiki retrying verification' do
expect(subject.count_wikis_retrying_verification).to eq 3
end
context 'with selective sync by namespace' do
it 'counts registries that wiki retrying verification where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_wikis_retrying_verification).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts registries that wiki retrying verification where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_wikis_retrying_verification).to eq 1
end
end
end
describe '#count_repositories_checksum_mismatch' do
before do
create(:geo_project_registry, :repository_checksum_mismatch, :wiki_checksum_mismatch, project: project)
create(:geo_project_registry, :repository_checksum_mismatch, project: project_1_in_synced_group)
create(:geo_project_registry, :repository_checksum_mismatch, :wiki_verified, project: project_4_broken_storage)
create(:geo_project_registry, :wiki_checksum_mismatch, project: project_2_in_synced_group)
end
it 'counts registries that repository mismatch' do
expect(subject.count_repositories_checksum_mismatch).to eq 3
end
context 'with selective sync by namespace' do
it 'counts mismatch registries where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_repositories_checksum_mismatch).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts mismatch registries where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_repositories_checksum_mismatch).to eq 1
end
end
end
describe '#count_wikis_checksum_mismatch' do
before do
create(:geo_project_registry, :repository_checksum_mismatch, :wiki_checksum_mismatch, project: project)
create(:geo_project_registry, :repository_checksum_mismatch, project: project_1_in_synced_group)
create(:geo_project_registry, :wiki_checksum_mismatch, project: project_2_in_synced_group)
create(:geo_project_registry, :repository_verified, :wiki_checksum_mismatch, project: project_4_broken_storage)
end
it 'counts registries that wiki mismatch' do
expect(subject.count_wikis_checksum_mismatch).to eq 3
end
context 'with selective sync by namespace' do
it 'counts mismatch registries where projects belongs to the namespaces' do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_wikis_checksum_mismatch).to eq 1
end
end
context 'with selective sync by shard' do
it 'counts mismatch registries where projects belongs to the shards' do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject.count_wikis_checksum_mismatch).to eq 1
end
end
end
end
shared_examples 'delegates to the proper finder' do |legacy_finder_klass, finder_klass, method, args|
where(:selective_sync, :fdw_enabled, :fdw_for_selective_sync, :finder) do
false | false | false | legacy_finder_klass
false | false | true | legacy_finder_klass
false | true | true | finder_klass
false | true | false | finder_klass
true | false | false | legacy_finder_klass
true | false | true | legacy_finder_klass
true | true | true | finder_klass
true | true | false | legacy_finder_klass
end
with_them do
before do
stub_fdw(fdw_enabled)
stub_feature_flags(use_fdw_queries_for_selective_sync: fdw_for_selective_sync)
stub_selective_sync(secondary, selective_sync)
end
it 'delegates to the proper finder' do
expect_next_instance_of(finder) do |finder|
expect(finder).to receive(:execute).once
end
subject.public_send(method, *args)
end
end
end
context 'FDW', :geo_fdw do
context 'with use_fdw_queries_for_selective_sync disabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: false)
end
include_examples 'counts all the things', 'fdw'
end
context 'with use_fdw_queries_for_selective_sync enabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: true)
end
include_examples 'counts all the things', 'fdw'
end
end
context 'Legacy' do
before do
stub_fdw_disabled
end
include_examples 'counts all the things', 'legacy'
end
describe '#find_unsynced_projects', :geo_fdw do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectUnsyncedFinder,
Geo::ProjectUnsyncedFinder,
:find_unsynced_projects, [shard_name: 'default', batch_size: 100]
end
describe '#find_projects_updated_recently', :geo_fdw do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectUpdatedRecentlyFinder,
Geo::ProjectUpdatedRecentlyFinder,
:find_projects_updated_recently, [shard_name: 'default', batch_size: 100]
end
describe '#find_failed_project_registries', :geo_fdw do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectRegistrySyncFailedFinder,
Geo::ProjectRegistrySyncFailedFinder,
:find_failed_project_registries, ['repository']
end
describe '#find_registries_to_verify', :geo_fdw do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectRegistryPendingVerificationFinder,
Geo::ProjectRegistryPendingVerificationFinder,
:find_registries_to_verify, [shard_name: 'default', batch_size: 100]
end
describe '#find_verification_failed_project_registries', :geo_fdw do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectRegistryVerificationFailedFinder,
Geo::ProjectRegistryVerificationFailedFinder,
:find_verification_failed_project_registries, ['repository']
end
describe '#find_checksum_mismatch_project_registries', :geo_fdw do
include_examples 'delegates to the proper finder',
Geo::LegacyProjectRegistryMismatchFinder,
Geo::ProjectRegistryMismatchFinder,
:find_checksum_mismatch_project_registries, ['repository']
end
end
ee/spec/models/geo_node_status_spec.rb
View file @ 7a06ed3f
......@@ -122,6 +122,12 @@ describe GeoNodeStatus, :geo, :geo_fdw do
end
end
describe '#projects_count' do
it 'counts the number of projects' do
expect(subject.projects_count).to eq 4
end
end
describe '#attachments_synced_count' do
it 'only counts successful syncs' do
create_list(:user, 3, avatar: fixture_file_upload('spec/fixtures/dk.png', 'image/png'))
......@@ -997,12 +1003,6 @@ describe GeoNodeStatus, :geo, :geo_fdw do
stub_current_geo_node(primary)
end
it 'does not call ProjectRegistryFinder#count_projects' do
expect_any_instance_of(Geo::ProjectRegistryFinder).not_to receive(:count_projects)
subject
end
it 'does not call LfsObjectRegistryFinder#count_syncable' do
expect_any_instance_of(Geo::LfsObjectRegistryFinder).not_to receive(:count_syncable)
......@@ -1023,12 +1023,6 @@ describe GeoNodeStatus, :geo, :geo_fdw do
end
context 'on the secondary' do
it 'calls ProjectRegistryFinder#count_projects' do
expect_any_instance_of(Geo::ProjectRegistryFinder).to receive(:count_projects)
subject
end
it 'calls LfsObjectRegistryFinder#count_syncable' do
expect_any_instance_of(Geo::AttachmentRegistryFinder).to receive(:count_syncable)
......
ee/spec/workers/geo/file_download_dispatch_worker_spec.rb
View file @ 7a06ed3f
require 'spec_helper'
describe Geo::FileDownloadDispatchWorker, :geo do
describe Geo::FileDownloadDispatchWorker, :geo, :geo_fdw do
include ::EE::GeoHelpers
include ExclusiveLeaseHelpers
......@@ -11,397 +11,384 @@ describe Geo::FileDownloadDispatchWorker, :geo do
stub_current_geo_node(secondary)
stub_exclusive_lease(renew: true)
allow_any_instance_of(described_class).to receive(:over_time?).and_return(false)
WebMock.stub_request(:get, /primary-geo-node/).to_return(status: 200, body: "", headers: {})
end
shared_examples '#perform' do
it 'does not schedule anything when tracking database is not configured' do
create(:lfs_object, :with_file)
it 'does not schedule anything when tracking database is not configured' do
create(:lfs_object, :with_file)
allow(Gitlab::Geo).to receive(:geo_database_configured?) { false }
allow(Gitlab::Geo).to receive(:geo_database_configured?) { false }
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
subject.perform
# We need to unstub here or the DatabaseCleaner will have issues since it
# will appear as though the tracking DB were not available
allow(Gitlab::Geo).to receive(:geo_database_configured?).and_call_original
end
it 'does not schedule anything when node is disabled' do
create(:lfs_object, :with_file)
secondary.enabled = false
secondary.save
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
subject.perform
end
context 'with attachments (Upload records)' do
let(:upload) { create(:upload) }
it 'performs Geo::FileDownloadWorker for unsynced attachments' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', upload.id)
subject.perform
end
subject.perform
it 'performs Geo::FileDownloadWorker for failed-sync attachments' do
create(:geo_file_registry, :avatar, :failed, file_id: upload.id, bytes: 0)
# We need to unstub here or the DatabaseCleaner will have issues since it
# will appear as though the tracking DB were not available
allow(Gitlab::Geo).to receive(:geo_database_configured?).and_call_original
end
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('avatar', upload.id).once.and_return(spy)
it 'does not schedule anything when node is disabled' do
create(:lfs_object, :with_file)
subject.perform
end
secondary.enabled = false
secondary.save
it 'does not perform Geo::FileDownloadWorker for synced attachments' do
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 1234)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
subject.perform
end
subject.perform
end
context 'with attachments (Upload records)' do
let(:upload) { create(:upload) }
it 'does not perform Geo::FileDownloadWorker for synced attachments even with 0 bytes downloaded' do
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 0)
it 'performs Geo::FileDownloadWorker for unsynced attachments' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', upload.id)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
subject.perform
end
subject.perform
end
it 'performs Geo::FileDownloadWorker for failed-sync attachments' do
create(:geo_file_registry, :avatar, :failed, file_id: upload.id, bytes: 0)
context 'with a failed file' do
let(:failed_registry) { create(:geo_file_registry, :avatar, :failed, file_id: 999) }
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('avatar', upload.id).once.and_return(spy)
it 'does not stall backfill' do
unsynced = create(:upload)
subject.perform
end
stub_const('Geo::Scheduler::SchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 1)
it 'does not perform Geo::FileDownloadWorker for synced attachments' do
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 1234)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('avatar', failed_registry.file_id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', unsynced.id)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
subject.perform
end
subject.perform
end
it 'retries failed files' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', failed_registry.file_id)
it 'does not perform Geo::FileDownloadWorker for synced attachments even with 0 bytes downloaded' do
create(:geo_file_registry, :avatar, file_id: upload.id, bytes: 0)
subject.perform
end
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
it 'does not retry failed files when retry_at is tomorrow' do
failed_registry = create(:geo_file_registry, :avatar, :failed, file_id: 999, retry_at: Date.tomorrow)
subject.perform
end
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('avatar', failed_registry.file_id)
context 'with a failed file' do
let(:failed_registry) { create(:geo_file_registry, :avatar, :failed, file_id: 999) }
subject.perform
end
it 'does not stall backfill' do
unsynced = create(:upload)
it 'retries failed files when retry_at is in the past' do
failed_registry = create(:geo_file_registry, :avatar, :failed, file_id: 999, retry_at: Date.yesterday)
stub_const('Geo::Scheduler::SchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 1)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', failed_registry.file_id)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('avatar', failed_registry.file_id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', unsynced.id)
subject.perform
end
subject.perform
end
context 'with Upload files missing on the primary that are marked as synced' do
let(:synced_upload_with_file_missing_on_primary) { create(:upload) }
before do
Geo::FileRegistry.create!(file_type: :avatar, file_id: synced_upload_with_file_missing_on_primary.id, bytes: 1234, success: true, missing_on_primary: true)
end
it 'retries the files if there is spare capacity' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', synced_upload_with_file_missing_on_primary.id)
it 'retries failed files' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', failed_registry.file_id)
subject.perform
end
subject.perform
end
it 'does not retry those files if there is no spare capacity' do
unsynced_upload = create(:upload)
expect(subject).to receive(:db_retrieve_batch_size).and_return(1).twice
it 'does not retry failed files when retry_at is tomorrow' do
failed_registry = create(:geo_file_registry, :avatar, :failed, file_id: 999, retry_at: Date.tomorrow)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', unsynced_upload.id)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('avatar', failed_registry.file_id)
subject.perform
end
subject.perform
end
it 'does not retry those files if they are already scheduled' do
unsynced_upload = create(:upload)
it 'retries failed files when retry_at is in the past' do
failed_registry = create(:geo_file_registry, :avatar, :failed, file_id: 999, retry_at: Date.yesterday)
scheduled_jobs = [{ type: 'avatar', id: synced_upload_with_file_missing_on_primary.id, job_id: 'foo' }]
expect(subject).to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(1)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', unsynced_upload.id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', failed_registry.file_id)
subject.perform
end
subject.perform
end
end
context 'with LFS objects' do
let!(:lfs_object_local_store) { create(:lfs_object, :with_file) }
let!(:lfs_object_remote_store) { create(:lfs_object, :with_file) }
context 'with Upload files missing on the primary that are marked as synced' do
let(:synced_upload_with_file_missing_on_primary) { create(:upload) }
before do
stub_lfs_object_storage
lfs_object_remote_store.file.migrate!(LfsObjectUploader::Store::REMOTE)
Geo::FileRegistry.create!(file_type: :avatar, file_id: synced_upload_with_file_missing_on_primary.id, bytes: 1234, success: true, missing_on_primary: true)
end
it 'filters S3-backed files' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_local_store.id)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('lfs', lfs_object_remote_store.id)
it 'retries the files if there is spare capacity' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', synced_upload_with_file_missing_on_primary.id)
subject.perform
end
context 'with files missing on the primary that are marked as synced' do
let!(:lfs_object_file_missing_on_primary) { create(:lfs_object, :with_file) }
before do
Geo::FileRegistry.create!(file_type: :lfs, file_id: lfs_object_file_missing_on_primary.id, bytes: 1234, success: true, missing_on_primary: true)
end
it 'retries the files if there is spare capacity' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_local_store.id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_file_missing_on_primary.id)
subject.perform
end
it 'does not retry those files if there is no spare capacity' do
expect(subject).to receive(:db_retrieve_batch_size).and_return(1).twice
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_local_store.id)
subject.perform
end
it 'does not retry those files if there is no spare capacity' do
unsynced_upload = create(:upload)
expect(subject).to receive(:db_retrieve_batch_size).and_return(1).twice
it 'does not retry those files if they are already scheduled' do
scheduled_jobs = [{ type: 'lfs', id: lfs_object_file_missing_on_primary.id, job_id: 'foo' }]
expect(subject).to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(1)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_local_store.id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', unsynced_upload.id)
subject.perform
end
subject.perform
end
end
context 'with job artifacts' do
it 'performs Geo::FileDownloadWorker for unsynced job artifacts' do
artifact = create(:ci_job_artifact)
it 'does not retry those files if they are already scheduled' do
unsynced_upload = create(:upload)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact.id)
scheduled_jobs = [{ type: 'avatar', id: synced_upload_with_file_missing_on_primary.id, job_id: 'foo' }]
expect(subject).to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(1)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('avatar', unsynced_upload.id)
subject.perform
end
end
end
it 'performs Geo::FileDownloadWorker for failed-sync job artifacts' do
artifact = create(:ci_job_artifact)
context 'with LFS objects' do
let!(:lfs_object_local_store) { create(:lfs_object, :with_file) }
let!(:lfs_object_remote_store) { create(:lfs_object, :with_file) }
create(:geo_job_artifact_registry, artifact_id: artifact.id, bytes: 0, success: false)
before do
stub_lfs_object_storage
lfs_object_remote_store.file.migrate!(LfsObjectUploader::Store::REMOTE)
end
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('job_artifact', artifact.id).once.and_return(spy)
it 'filters S3-backed files' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_local_store.id)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('lfs', lfs_object_remote_store.id)
subject.perform
end
subject.perform
end
it 'does not perform Geo::FileDownloadWorker for synced job artifacts' do
artifact = create(:ci_job_artifact)
context 'with files missing on the primary that are marked as synced' do
let!(:lfs_object_file_missing_on_primary) { create(:lfs_object, :with_file) }
create(:geo_job_artifact_registry, artifact_id: artifact.id, bytes: 1234, success: true)
before do
Geo::FileRegistry.create!(file_type: :lfs, file_id: lfs_object_file_missing_on_primary.id, bytes: 1234, success: true, missing_on_primary: true)
end
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
it 'retries the files if there is spare capacity' do
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_local_store.id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_file_missing_on_primary.id)
subject.perform
end
it 'does not perform Geo::FileDownloadWorker for synced job artifacts even with 0 bytes downloaded' do
artifact = create(:ci_job_artifact)
create(:geo_job_artifact_registry, artifact_id: artifact.id, bytes: 0, success: true)
it 'does not retry those files if there is no spare capacity' do
expect(subject).to receive(:db_retrieve_batch_size).and_return(1).twice
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_local_store.id)
subject.perform
end
it 'does not retry failed artifacts when retry_at is tomorrow' do
failed_registry = create(:geo_job_artifact_registry, :with_artifact, bytes: 0, success: false, retry_at: Date.tomorrow)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('job_artifact', failed_registry.artifact_id)
it 'does not retry those files if they are already scheduled' do
scheduled_jobs = [{ type: 'lfs', id: lfs_object_file_missing_on_primary.id, job_id: 'foo' }]
expect(subject).to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(1)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('lfs', lfs_object_local_store.id)
subject.perform
end
end
end
it 'retries failed artifacts when retry_at is in the past' do
failed_registry = create(:geo_job_artifact_registry, :with_artifact, success: false, retry_at: Date.yesterday)
context 'with job artifacts' do
it 'performs Geo::FileDownloadWorker for unsynced job artifacts' do
artifact = create(:ci_job_artifact)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', failed_registry.artifact_id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact.id)
subject.perform
end
context 'with files missing on the primary that are marked as synced' do
let!(:artifact_file_missing_on_primary) { create(:ci_job_artifact) }
let!(:artifact_registry) { create(:geo_job_artifact_registry, artifact_id: artifact_file_missing_on_primary.id, bytes: 1234, success: true, missing_on_primary: true) }
subject.perform
end
it 'retries the files if there is spare capacity' do
artifact = create(:ci_job_artifact)
it 'performs Geo::FileDownloadWorker for failed-sync job artifacts' do
artifact = create(:ci_job_artifact)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact.id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact_file_missing_on_primary.id)
create(:geo_job_artifact_registry, artifact_id: artifact.id, bytes: 0, success: false)
subject.perform
end
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('job_artifact', artifact.id).once.and_return(spy)
it 'retries failed files with retry_at in the past' do
artifact_registry.update!(retry_at: Date.yesterday)
subject.perform
end
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact_file_missing_on_primary.id)
it 'does not perform Geo::FileDownloadWorker for synced job artifacts' do
artifact = create(:ci_job_artifact)
subject.perform
end
create(:geo_job_artifact_registry, artifact_id: artifact.id, bytes: 1234, success: true)
it 'does not retry files with later retry_at' do
artifact_registry.update!(retry_at: Date.tomorrow)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('job_artifact', artifact_file_missing_on_primary.id)
subject.perform
end
subject.perform
end
it 'does not perform Geo::FileDownloadWorker for synced job artifacts even with 0 bytes downloaded' do
artifact = create(:ci_job_artifact)
it 'does not retry those files if there is no spare capacity' do
artifact = create(:ci_job_artifact)
create(:geo_job_artifact_registry, artifact_id: artifact.id, bytes: 0, success: true)
expect(subject).to receive(:db_retrieve_batch_size).and_return(1).twice
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact.id)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async)
subject.perform
end
subject.perform
end
it 'does not retry those files if they are already scheduled' do
artifact = create(:ci_job_artifact)
it 'does not retry failed artifacts when retry_at is tomorrow' do
failed_registry = create(:geo_job_artifact_registry, :with_artifact, bytes: 0, success: false, retry_at: Date.tomorrow)
scheduled_jobs = [{ type: 'job_artifact', id: artifact_file_missing_on_primary.id, job_id: 'foo' }]
expect(subject).to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(1)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact.id)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('job_artifact', failed_registry.artifact_id)
subject.perform
end
end
subject.perform
end
context 'backoff time' do
let(:cache_key) { "#{described_class.name.underscore}:skip" }
it 'retries failed artifacts when retry_at is in the past' do
failed_registry = create(:geo_job_artifact_registry, :with_artifact, success: false, retry_at: Date.yesterday)
it 'does not set the back off time when there are no pending items' do
expect(Rails.cache).not_to receive(:write).with(cache_key, true, expires_in: 300.seconds)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', failed_registry.artifact_id)
subject.perform
end
subject.perform
end
# Test the case where we have:
#
# 1. A total of 10 files in the queue, and we can load a maximimum of 5 and send 2 at a time.
# 2. We send 2, wait for 1 to finish, and then send again.
it 'attempts to load a new batch without pending downloads' do
stub_const('Geo::Scheduler::SchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 5)
secondary.update!(files_max_capacity: 2)
result_object = double(:result, success: true, bytes_downloaded: 100, primary_missing_file: false)
allow_any_instance_of(::Gitlab::Geo::Transfer).to receive(:download_from_primary).and_return(result_object)
context 'with files missing on the primary that are marked as synced' do
let!(:artifact_file_missing_on_primary) { create(:ci_job_artifact) }
let!(:artifact_registry) { create(:geo_job_artifact_registry, artifact_id: artifact_file_missing_on_primary.id, bytes: 1234, success: true, missing_on_primary: true) }
it 'retries the files if there is spare capacity' do
artifact = create(:ci_job_artifact)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact.id)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact_file_missing_on_primary.id)
avatar = fixture_file_upload('spec/fixtures/dk.png')
create_list(:lfs_object, 2, :with_file)
create_list(:user, 2, avatar: avatar)
create_list(:note, 2, :with_attachment)
create_list(:upload, 1, :personal_snippet_upload)
create_list(:ci_job_artifact, 1)
create(:appearance, logo: avatar, header_logo: avatar)
expect(Geo::FileDownloadWorker).to receive(:perform_async).exactly(10).times.and_call_original
# For 10 downloads, we expect four database reloads:
# 1. Load the first batch of 5.
# 2. 4 get sent out, 1 remains. This triggers another reload, which loads in the next 5.
# 3. Those 4 get sent out, and 1 remains.
# 3. Since the second reload filled the pipe with 4, we need to do a final reload to ensure
# zero are left.
expect(subject).to receive(:load_pending_resources).exactly(4).times.and_call_original
Sidekiq::Testing.inline! do
subject.perform
end
end
context 'when node has namespace restrictions', :request_store do
let(:synced_group) { create(:group) }
let(:project_in_synced_group) { create(:project, group: synced_group) }
let(:unsynced_project) { create(:project) }
it 'retries failed files with retry_at in the past' do
artifact_registry.update!(retry_at: Date.yesterday)
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact_file_missing_on_primary.id)
allow(ProjectCacheWorker).to receive(:perform_async).and_return(true)
allow(::Gitlab::Geo).to receive(:current_node).and_call_original
Rails.cache.write(:current_node, secondary.to_json)
allow(::GeoNode).to receive(:current_node).and_return(secondary)
subject.perform
end
it 'does not perform Geo::FileDownloadWorker for LFS object that does not belong to selected namespaces to replicate' do
lfs_object_in_synced_group = create(:lfs_objects_project, project: project_in_synced_group)
create(:lfs_objects_project, project: unsynced_project)
it 'does not retry files with later retry_at' do
artifact_registry.update!(retry_at: Date.tomorrow)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('lfs', lfs_object_in_synced_group.lfs_object_id).once.and_return(spy)
expect(Geo::FileDownloadWorker).not_to receive(:perform_async).with('job_artifact', artifact_file_missing_on_primary.id)
subject.perform
end
it 'does not perform Geo::FileDownloadWorker for job artifact that does not belong to selected namespaces to replicate' do
create(:ci_job_artifact, project: unsynced_project)
job_artifact_in_synced_group = create(:ci_job_artifact, project: project_in_synced_group)
it 'does not retry those files if there is no spare capacity' do
artifact = create(:ci_job_artifact)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('job_artifact', job_artifact_in_synced_group.id).once.and_return(spy)
expect(subject).to receive(:db_retrieve_batch_size).and_return(1).twice
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact.id)
subject.perform
end
it 'does not perform Geo::FileDownloadWorker for upload objects that do not belong to selected namespaces to replicate' do
avatar = fixture_file_upload('spec/fixtures/dk.png')
avatar_in_synced_group = create(:upload, model: synced_group, path: avatar)
create(:upload, model: create(:group), path: avatar)
avatar_in_project_in_synced_group = create(:upload, model: project_in_synced_group, path: avatar)
create(:upload, model: unsynced_project, path: avatar)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('avatar', avatar_in_project_in_synced_group.id).once.and_return(spy)
it 'does not retry those files if they are already scheduled' do
artifact = create(:ci_job_artifact)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('avatar', avatar_in_synced_group.id).once.and_return(spy)
scheduled_jobs = [{ type: 'job_artifact', id: artifact_file_missing_on_primary.id, job_id: 'foo' }]
expect(subject).to receive(:scheduled_jobs).and_return(scheduled_jobs).at_least(1)
expect(Geo::FileDownloadWorker).to receive(:perform_async).with('job_artifact', artifact.id)
subject.perform
end
end
end
describe 'when PostgreSQL FDW is available', :geo_fdw do
it_behaves_like '#perform'
context 'backoff time' do
let(:cache_key) { "#{described_class.name.underscore}:skip" }
it 'does not set the back off time when there are no pending items' do
expect(Rails.cache).not_to receive(:write).with(cache_key, true, expires_in: 300.seconds)
subject.perform
end
end
# Test the case where we have:
#
# 1. A total of 10 files in the queue, and we can load a maximimum of 5 and send 2 at a time.
# 2. We send 2, wait for 1 to finish, and then send again.
it 'attempts to load a new batch without pending downloads' do
stub_const('Geo::Scheduler::SchedulerWorker::DB_RETRIEVE_BATCH_SIZE', 5)
secondary.update!(files_max_capacity: 2)
result_object = double(:result, success: true, bytes_downloaded: 100, primary_missing_file: false)
allow_any_instance_of(::Gitlab::Geo::Transfer).to receive(:download_from_primary).and_return(result_object)
avatar = fixture_file_upload('spec/fixtures/dk.png')
create_list(:lfs_object, 2, :with_file)
create_list(:user, 2, avatar: avatar)
create_list(:note, 2, :with_attachment)
create(:upload, :personal_snippet_upload)
create(:ci_job_artifact)
create(:appearance, logo: avatar, header_logo: avatar)
expect(Geo::FileDownloadWorker).to receive(:perform_async).exactly(10).times.and_call_original
# For 10 downloads, we expect four database reloads:
# 1. Load the first batch of 5.
# 2. 4 get sent out, 1 remains. This triggers another reload, which loads in the next 5.
# 3. Those 4 get sent out, and 1 remains.
# 3. Since the second reload filled the pipe with 4, we need to do a final reload to ensure
# zero are left.
expect(subject).to receive(:load_pending_resources).exactly(4).times.and_call_original
Sidekiq::Testing.inline! do
subject.perform
end
end
describe 'when PostgreSQL FDW is not enabled' do
context 'when node has namespace restrictions', :request_store do
let(:synced_group) { create(:group) }
let(:project_in_synced_group) { create(:project, group: synced_group) }
let(:unsynced_project) { create(:project) }
before do
allow(Gitlab::Geo::Fdw).to receive(:enabled?).and_return(false)
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
allow(ProjectCacheWorker).to receive(:perform_async).and_return(true)
allow(::Gitlab::Geo).to receive(:current_node).and_call_original
Rails.cache.write(:current_node, secondary.to_json)
allow(::GeoNode).to receive(:current_node).and_return(secondary)
end
it_behaves_like '#perform'
it 'does not perform Geo::FileDownloadWorker for LFS object that does not belong to selected namespaces to replicate' do
lfs_object_in_synced_group = create(:lfs_objects_project, project: project_in_synced_group)
create(:lfs_objects_project, project: unsynced_project)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('lfs', lfs_object_in_synced_group.lfs_object_id).once.and_return(spy)
subject.perform
end
it 'does not perform Geo::FileDownloadWorker for job artifact that does not belong to selected namespaces to replicate' do
create(:ci_job_artifact, project: unsynced_project)
job_artifact_in_synced_group = create(:ci_job_artifact, project: project_in_synced_group)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('job_artifact', job_artifact_in_synced_group.id).once.and_return(spy)
subject.perform
end
it 'does not perform Geo::FileDownloadWorker for upload objects that do not belong to selected namespaces to replicate' do
avatar = fixture_file_upload('spec/fixtures/dk.png')
avatar_in_synced_group = create(:upload, model: synced_group, path: avatar)
create(:upload, model: create(:group), path: avatar)
avatar_in_project_in_synced_group = create(:upload, model: project_in_synced_group, path: avatar)
create(:upload, model: unsynced_project, path: avatar)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('avatar', avatar_in_project_in_synced_group.id).once.and_return(spy)
expect(Geo::FileDownloadWorker).to receive(:perform_async)
.with('avatar', avatar_in_synced_group.id).once.and_return(spy)
subject.perform
end
end
end
ee/spec/workers/geo/migrated_local_files_clean_up_worker_spec.rb
View file @ 7a06ed3f
require 'spec_helper'
describe Geo::MigratedLocalFilesCleanUpWorker, :geo do
describe Geo::MigratedLocalFilesCleanUpWorker, :geo, :geo_fdw do
include ::EE::GeoHelpers
include ExclusiveLeaseHelpers
let(:primary) { create(:geo_node, :primary, host: 'primary-geo-node') }
let(:secondary) { create(:geo_node) }
subject(:worker) { described_class.new }
before do
stub_current_geo_node(secondary)
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:try_obtain).and_return(true)
allow_any_instance_of(Gitlab::ExclusiveLease).to receive(:renew).and_return(true)
stub_exclusive_lease(renew: true)
end
shared_examples '#perform' do
it 'does not run when node is disabled' do
secondary.enabled = false
secondary.save
it 'does not run when node is disabled' do
secondary.update_column(:enabled, false)
expect(worker).not_to receive(:try_obtain_lease)
expect(subject).not_to receive(:try_obtain_lease)
worker.perform
end
subject.perform
end
context 'with LFS objects' do
let(:lfs_object_local) { create(:lfs_object) }
let(:lfs_object_remote) { create(:lfs_object, :object_storage) }
context 'with LFS objects' do
let(:lfs_object_local) { create(:lfs_object) }
let(:lfs_object_remote) { create(:lfs_object, :object_storage) }
before do
stub_lfs_object_storage
before do
stub_lfs_object_storage
create(:geo_file_registry, :lfs, file_id: lfs_object_local.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_remote.id)
end
create(:geo_file_registry, :lfs, file_id: lfs_object_local.id)
create(:geo_file_registry, :lfs, file_id: lfs_object_remote.id)
end
it 'schedules job for file stored remotely and synced locally' do
expect(worker).to receive(:schedule_job).with('lfs', lfs_object_remote.id)
expect(worker).not_to receive(:schedule_job).with(anything, lfs_object_local.id)
it 'schedules job for file stored remotely and synced locally' do
expect(subject).to receive(:schedule_job).with('lfs', lfs_object_remote.id)
expect(subject).not_to receive(:schedule_job).with(anything, lfs_object_local.id)
worker.perform
end
subject.perform
end
it 'schedules worker for file stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('lfs', lfs_object_remote.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_local.id)
it 'schedules worker for file stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('lfs', lfs_object_remote.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_local.id)
worker.perform
end
subject.perform
end
end
context 'with attachments' do
let(:avatar_upload) { create(:upload) }
let(:personal_snippet_upload) { create(:upload, :personal_snippet_upload) }
let(:issuable_upload) { create(:upload, :issuable_upload) }
let(:namespace_upload) { create(:upload, :namespace_upload) }
let(:attachment_upload) { create(:upload, :attachment_upload) }
let(:favicon_upload) { create(:upload, :favicon_upload) }
context 'with attachments' do
let(:avatar_upload) { create(:upload) }
let(:personal_snippet_upload) { create(:upload, :personal_snippet_upload) }
let(:issuable_upload) { create(:upload, :issuable_upload) }
let(:namespace_upload) { create(:upload, :namespace_upload) }
let(:attachment_upload) { create(:upload, :attachment_upload) }
let(:favicon_upload) { create(:upload, :favicon_upload) }
before do
create(:geo_file_registry, :avatar, file_id: avatar_upload.id)
create(:geo_file_registry, :personal_file, file_id: personal_snippet_upload.id)
create(:geo_file_registry, :file, file_id: issuable_upload.id)
create(:geo_file_registry, :namespace_file, file_id: namespace_upload.id)
create(:geo_file_registry, :attachment, file_id: attachment_upload.id)
create(:geo_file_registry, :favicon, file_id: favicon_upload.id)
end
it 'schedules nothing for attachments stored locally' do
expect(worker).not_to receive(:schedule_job).with(anything, avatar_upload.id)
expect(worker).not_to receive(:schedule_job).with(anything, personal_snippet_upload.id)
expect(worker).not_to receive(:schedule_job).with(anything, issuable_upload.id)
expect(worker).not_to receive(:schedule_job).with(anything, namespace_upload.id)
expect(worker).not_to receive(:schedule_job).with(anything, attachment_upload.id)
expect(worker).not_to receive(:schedule_job).with(anything, favicon_upload.id)
before do
create(:geo_file_registry, :avatar, file_id: avatar_upload.id)
create(:geo_file_registry, :personal_file, file_id: personal_snippet_upload.id)
create(:geo_file_registry, :file, file_id: issuable_upload.id)
create(:geo_file_registry, :namespace_file, file_id: namespace_upload.id)
create(:geo_file_registry, :attachment, file_id: attachment_upload.id)
create(:geo_file_registry, :favicon, file_id: favicon_upload.id)
end
worker.perform
end
it 'schedules nothing for attachments stored locally' do
expect(subject).not_to receive(:schedule_job).with(anything, avatar_upload.id)
expect(subject).not_to receive(:schedule_job).with(anything, personal_snippet_upload.id)
expect(subject).not_to receive(:schedule_job).with(anything, issuable_upload.id)
expect(subject).not_to receive(:schedule_job).with(anything, namespace_upload.id)
expect(subject).not_to receive(:schedule_job).with(anything, attachment_upload.id)
expect(subject).not_to receive(:schedule_job).with(anything, favicon_upload.id)
context 'attachments stored remotely' do
before do
stub_uploads_object_storage(AvatarUploader)
stub_uploads_object_storage(PersonalFileUploader)
stub_uploads_object_storage(FileUploader)
stub_uploads_object_storage(NamespaceFileUploader)
stub_uploads_object_storage(AttachmentUploader)
stub_uploads_object_storage(FaviconUploader)
avatar_upload.update_column(:store, FileUploader::Store::REMOTE)
personal_snippet_upload.update_column(:store, FileUploader::Store::REMOTE)
issuable_upload.update_column(:store, FileUploader::Store::REMOTE)
namespace_upload.update_column(:store, FileUploader::Store::REMOTE)
attachment_upload.update_column(:store, FileUploader::Store::REMOTE)
favicon_upload.update_column(:store, FileUploader::Store::REMOTE)
end
it 'schedules jobs for uploads stored remotely and synced locally' do
expect(worker).to receive(:schedule_job).with('avatar', avatar_upload.id)
expect(worker).to receive(:schedule_job).with('personal_file', personal_snippet_upload.id)
expect(worker).to receive(:schedule_job).with('file', issuable_upload.id)
expect(worker).to receive(:schedule_job).with('namespace_file', namespace_upload.id)
expect(worker).to receive(:schedule_job).with('attachment', attachment_upload.id)
expect(worker).to receive(:schedule_job).with('favicon', favicon_upload.id)
worker.perform
end
it 'schedules workers for uploads stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('avatar', avatar_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('personal_file', personal_snippet_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('file', issuable_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('namespace_file', namespace_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('attachment', attachment_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('favicon', favicon_upload.id)
worker.perform
end
end
subject.perform
end
context 'with job artifacts' do
let(:job_artifact_local) { create(:ci_job_artifact) }
let(:job_artifact_remote) { create(:ci_job_artifact, :remote_store) }
context 'attachments stored remotely' do
before do
stub_artifacts_object_storage
create(:geo_job_artifact_registry, artifact_id: job_artifact_local.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote.id)
stub_uploads_object_storage(AvatarUploader)
stub_uploads_object_storage(PersonalFileUploader)
stub_uploads_object_storage(FileUploader)
stub_uploads_object_storage(NamespaceFileUploader)
stub_uploads_object_storage(AttachmentUploader)
stub_uploads_object_storage(FaviconUploader)
avatar_upload.update_column(:store, FileUploader::Store::REMOTE)
personal_snippet_upload.update_column(:store, FileUploader::Store::REMOTE)
issuable_upload.update_column(:store, FileUploader::Store::REMOTE)
namespace_upload.update_column(:store, FileUploader::Store::REMOTE)
attachment_upload.update_column(:store, FileUploader::Store::REMOTE)
favicon_upload.update_column(:store, FileUploader::Store::REMOTE)
end
it 'schedules job for artifact stored remotely and synced locally' do
expect(worker).to receive(:schedule_job).with('job_artifact', job_artifact_remote.id)
expect(worker).not_to receive(:schedule_job).with(anything, job_artifact_local.id)
it 'schedules jobs for uploads stored remotely and synced locally' do
expect(subject).to receive(:schedule_job).with('avatar', avatar_upload.id)
expect(subject).to receive(:schedule_job).with('personal_file', personal_snippet_upload.id)
expect(subject).to receive(:schedule_job).with('file', issuable_upload.id)
expect(subject).to receive(:schedule_job).with('namespace_file', namespace_upload.id)
expect(subject).to receive(:schedule_job).with('attachment', attachment_upload.id)
expect(subject).to receive(:schedule_job).with('favicon', favicon_upload.id)
worker.perform
subject.perform
end
it 'schedules worker for artifact stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('job_artifact', job_artifact_remote.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, job_artifact_local.id)
it 'schedules workers for uploads stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('avatar', avatar_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('personal_file', personal_snippet_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('file', issuable_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('namespace_file', namespace_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('attachment', attachment_upload.id)
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('favicon', favicon_upload.id)
worker.perform
subject.perform
end
end
end
context 'backoff time' do
let(:cache_key) { "#{described_class.name.underscore}:skip" }
before do
stub_uploads_object_storage(AvatarUploader)
end
context 'with job artifacts' do
let(:job_artifact_local) { create(:ci_job_artifact) }
let(:job_artifact_remote) { create(:ci_job_artifact, :remote_store) }
it 'sets the back off time when there are no pending items' do
expect(Rails.cache).to receive(:write).with(cache_key, true, expires_in: 300.seconds).once
before do
stub_artifacts_object_storage
subject.perform
end
create(:geo_job_artifact_registry, artifact_id: job_artifact_local.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote.id)
end
it 'does not perform Geo::FileRegistryRemovalWorker when the backoff time is set' do
create(:geo_file_registry, :avatar)
it 'schedules job for artifact stored remotely and synced locally' do
expect(subject).to receive(:schedule_job).with('job_artifact', job_artifact_remote.id)
expect(subject).not_to receive(:schedule_job).with(anything, job_artifact_local.id)
expect(Rails.cache).to receive(:read).with(cache_key).and_return(true)
subject.perform
end
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async)
it 'schedules worker for artifact stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('job_artifact', job_artifact_remote.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, job_artifact_local.id)
subject.perform
end
subject.perform
end
end
describe 'when PostgreSQL FDW is available', :geo, :geo_fdw do
it_behaves_like '#perform'
end
context 'backoff time' do
let(:cache_key) { "#{described_class.name.underscore}:skip" }
describe 'when PostgreSQL FDW is not enabled', :geo do
before do
allow(Gitlab::Geo::Fdw).to receive(:enabled?).and_return(false)
stub_uploads_object_storage(AvatarUploader)
end
it 'sets the back off time when there are no pending items' do
expect(Rails.cache).to receive(:write).with(cache_key, true, expires_in: 300.seconds).once
subject.perform
end
it_behaves_like '#perform'
it 'does not perform Geo::FileRegistryRemovalWorker when the backoff time is set' do
create(:geo_file_registry, :avatar)
expect(Rails.cache).to receive(:read).with(cache_key).and_return(true)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async)
subject.perform
end
end
end
ee/spec/workers/geo/repository_verification/secondary/shard_worker_spec.rb
View file @ 7a06ed3f
require 'spec_helper'
describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clean_gitlab_redis_cache do
describe Geo::RepositoryVerification::Secondary::ShardWorker, :geo, :geo_fdw, :clean_gitlab_redis_cache do
include ::EE::GeoHelpers
include ExclusiveLeaseHelpers
let!(:secondary) { create(:geo_node) }
let(:shard_name) { Gitlab.config.repositories.storages.keys.first }
let(:secondary_singleworker) { Geo::RepositoryVerification::Secondary::SingleWorker }
set(:project) { create(:project) }
let(:secondary_single_worker) { Geo::RepositoryVerification::Secondary::SingleWorker }
let!(:project) { create(:project) }
before do
stub_current_geo_node(secondary)
......@@ -37,7 +36,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
create(:geo_project_registry, :synced, :repository_verification_outdated, project: project)
create(:geo_project_registry, :synced, :repository_verification_outdated, project: other_project)
expect(secondary_singleworker).to receive(:perform_async).twice
expect(secondary_single_worker).to receive(:perform_async).twice
subject.perform(shard_name)
end
......@@ -46,7 +45,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
create(:repository_state, :repository_verified, :wiki_verified, project: project)
missing_repository_verification = create(:geo_project_registry, :synced, :wiki_verified, project: project)
expect(secondary_singleworker).to receive(:perform_async).with(missing_repository_verification.id)
expect(secondary_single_worker).to receive(:perform_async).with(missing_repository_verification.id)
subject.perform(shard_name)
end
......@@ -55,7 +54,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
create(:repository_state, :repository_verified, :wiki_verified, project: project)
missing_wiki_verification = create(:geo_project_registry, :synced, :repository_verified, project: project)
expect(secondary_singleworker).to receive(:perform_async).with(missing_wiki_verification.id)
expect(secondary_single_worker).to receive(:perform_async).with(missing_wiki_verification.id)
subject.perform(shard_name)
end
......@@ -66,7 +65,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
create(:repository_state, :repository_verified, :wiki_verified, project: project_other_shard)
registry_other_shard = create(:geo_project_registry, :synced, :wiki_verified, project: project_other_shard)
expect(secondary_singleworker).not_to receive(:perform_async).with(registry_other_shard.id)
expect(secondary_single_worker).not_to receive(:perform_async).with(registry_other_shard.id)
subject.perform(shard_name)
end
......@@ -78,7 +77,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
create(:geo_project_registry, :synced, project: project, repository_missing_on_primary: true)
create(:geo_project_registry, :synced, project: other_project, wiki_missing_on_primary: true)
expect(secondary_singleworker).not_to receive(:perform_async)
expect(secondary_single_worker).not_to receive(:perform_async)
subject.perform(shard_name)
end
......@@ -101,13 +100,13 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
reg1 = create(:geo_project_registry, :synced, :repository_verification_outdated, project: project1_repo_verified)
reg2 = create(:geo_project_registry, :synced, :repository_verification_outdated, project: project2_repo_verified)
expect(secondary_singleworker).to receive(:perform_async).with(reg1.id).once
expect(secondary_single_worker).to receive(:perform_async).with(reg1.id).once
subject.perform(shard_name)
reg1.update!(repository_verification_checksum_sha: project1_repo_verified.repository_state.repository_verification_checksum)
expect(secondary_singleworker).to receive(:perform_async).with(reg2.id).once
expect(secondary_single_worker).to receive(:perform_async).with(reg2.id).once
subject.perform(shard_name)
end
......@@ -120,25 +119,25 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
create(:geo_project_registry, :synced, :repository_verification_failed, :wiki_verification_failed, project: project5_both_verified)
reg6 = create(:geo_project_registry, :synced, project: project6_both_verified)
expect(secondary_singleworker).to receive(:perform_async).with(reg1.id).once
expect(secondary_single_worker).to receive(:perform_async).with(reg1.id).once
subject.perform(shard_name)
reg1.update!(repository_verification_checksum_sha: project1_repo_verified.repository_state.repository_verification_checksum)
expect(secondary_singleworker).to receive(:perform_async).with(reg2.id).once
expect(secondary_single_worker).to receive(:perform_async).with(reg2.id).once
subject.perform(shard_name)
reg2.update!(repository_verification_checksum_sha: project2_repo_verified.repository_state.repository_verification_checksum)
expect(secondary_singleworker).to receive(:perform_async).with(reg4.id).once
expect(secondary_single_worker).to receive(:perform_async).with(reg4.id).once
subject.perform(shard_name)
reg4.update!(last_wiki_verification_failure: 'Failed!')
expect(secondary_singleworker).to receive(:perform_async).with(reg6.id).once
expect(secondary_single_worker).to receive(:perform_async).with(reg6.id).once
subject.perform(shard_name)
end
......@@ -149,7 +148,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
Gitlab::ShardHealthCache.update([])
expect(secondary_singleworker).not_to receive(:perform_async)
expect(secondary_single_worker).not_to receive(:perform_async)
subject.perform(shard_name)
end
......@@ -157,7 +156,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
it 'does not schedule jobs when no geo database is configured' do
allow(Gitlab::Geo).to receive(:geo_database_configured?) { false }
expect(secondary_singleworker).not_to receive(:perform_async)
expect(secondary_single_worker).not_to receive(:perform_async)
subject.perform(shard_name)
......@@ -169,7 +168,7 @@ describe Geo::RepositoryVerification::Secondary::ShardWorker, :postgresql, :clea
it 'does not schedule jobs when not running on a secondary' do
allow(Gitlab::Geo).to receive(:primary?) { false }
expect(secondary_singleworker).not_to receive(:perform_async)
expect(secondary_single_worker).not_to receive(:perform_async)
subject.perform(shard_name)
end
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7