From 1c5e99acaf78f7a951f0623fef927bb0ae7a1955 Mon Sep 17 00:00:00 2001 From: Douglas Barbosa Alexandre <dbalexandre@gmail.com> Date: Fri, 24 Jul 2020 15:53:20 -0300 Subject: [PATCH] Refactor endpoint to return failures for the current Geo node Since GitLab 13.2 we don't rely on FDW queries to sync/verify these repositories. --- .../geo/project_registry_mismatch_finder.rb | 31 ------ .../project_registry_sync_failed_finder.rb | 31 ------ ...ect_registry_verification_failed_finder.rb | 31 ------ ee/lib/api/geo_nodes.rb | 25 ++--- .../project_registry_mismatch_finder_spec.rb | 101 ------------------ ...roject_registry_sync_failed_finder_spec.rb | 101 ------------------ ...egistry_verification_failed_finder_spec.rb | 101 ------------------ 7 files changed, 13 insertions(+), 408 deletions(-) delete mode 100644 ee/app/finders/geo/project_registry_mismatch_finder.rb delete mode 100644 ee/app/finders/geo/project_registry_sync_failed_finder.rb delete mode 100644 ee/app/finders/geo/project_registry_verification_failed_finder.rb delete mode 100644 ee/spec/finders/geo/project_registry_mismatch_finder_spec.rb delete mode 100644 ee/spec/finders/geo/project_registry_sync_failed_finder_spec.rb delete mode 100644 ee/spec/finders/geo/project_registry_verification_failed_finder_spec.rb diff --git a/ee/app/finders/geo/project_registry_mismatch_finder.rb b/ee/app/finders/geo/project_registry_mismatch_finder.rb deleted file mode 100644 index 700b69ca038..00000000000 --- a/ee/app/finders/geo/project_registry_mismatch_finder.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -# Finder for retrieving project registries that checksum mismatch -# scoped to a type (repository or wiki) using FDW queries. -# -# Basic usage: -# -# Geo::ProjectRegistryMismatchFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute -# -# Valid `type` values are: -# -# * `:repository` -# * `:wiki` -# -# Any other value will be ignored. -module Geo - class ProjectRegistryMismatchFinder - def initialize(current_node:, type:) - @current_node = Geo::Fdw::GeoNode.find(current_node.id) - @type = type.to_s.to_sym - end - - def execute - current_node.project_registries.mismatch(type) - end - - private - - attr_reader :current_node, :type - end -end diff --git a/ee/app/finders/geo/project_registry_sync_failed_finder.rb b/ee/app/finders/geo/project_registry_sync_failed_finder.rb deleted file mode 100644 index 16ccd55f187..00000000000 --- a/ee/app/finders/geo/project_registry_sync_failed_finder.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -# Finder for retrieving project registries that synchronization have -# failed scoped to a type (repository or wiki) using FDW queries. -# -# Basic usage: -# -# Geo::ProjectRegistrySyncFailedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute -# -# Valid `type` values are: -# -# * `:repository` -# * `:wiki` -# -# Any other value will be ignored. -module Geo - class ProjectRegistrySyncFailedFinder - def initialize(current_node:, type:) - @current_node = Geo::Fdw::GeoNode.find(current_node.id) - @type = type.to_s.to_sym - end - - def execute - current_node.project_registries.sync_failed(type) - end - - private - - attr_reader :current_node, :type - end -end diff --git a/ee/app/finders/geo/project_registry_verification_failed_finder.rb b/ee/app/finders/geo/project_registry_verification_failed_finder.rb deleted file mode 100644 index 552bea3385d..00000000000 --- a/ee/app/finders/geo/project_registry_verification_failed_finder.rb +++ /dev/null @@ -1,31 +0,0 @@ -# frozen_string_literal: true - -# Finder for retrieving project registries that have verification failed -# scoped to a type (repository or wiki) using FDW queries. -# -# Basic usage: -# -# Geo::ProjectRegistryVerificationFailedFinder.new(current_node: Gitlab::Geo.current_node, :repository).execute -# -# Valid `type` values are: -# -# * `:repository` -# * `:wiki` -# -# Any other value will be ignored. -module Geo - class ProjectRegistryVerificationFailedFinder - def initialize(current_node:, type:) - @current_node = Geo::Fdw::GeoNode.find(current_node.id) - @type = type.to_s.to_sym - end - - def execute - current_node.project_registries.verification_failed(type) - end - - private - - attr_reader :current_node, :type - end -end diff --git a/ee/lib/api/geo_nodes.rb b/ee/lib/api/geo_nodes.rb index 1ffb7237b79..2c0b3c7a2c9 100644 --- a/ee/lib/api/geo_nodes.rb +++ b/ee/lib/api/geo_nodes.rb @@ -106,18 +106,19 @@ module API not_found!('Geo node not found') unless Gitlab::Geo.current_node forbidden!('Failures can only be requested from a secondary node') unless Gitlab::Geo.current_node.secondary? - finder_klass = case params[:failure_type] - when 'sync' - ::Geo::ProjectRegistrySyncFailedFinder - when 'verification' - ::Geo::ProjectRegistryVerificationFailedFinder - when 'checksum_mismatch' - ::Geo::ProjectRegistryMismatchFinder - else - not_found!('Failure type unknown') - end - - project_registries = finder_klass.new(current_node: Gitlab::Geo.current_node, type: params[:type]).execute + type = params[:type].to_s.to_sym + + project_registries = + case params[:failure_type] + when 'sync' + ::Geo::ProjectRegistry.sync_failed(type) + when 'verification' + ::Geo::ProjectRegistry.verification_failed(type) + when 'checksum_mismatch' + ::Geo::ProjectRegistry.mismatch(type) + else + not_found!('Failure type unknown') + end present paginate(project_registries), with: ::GeoProjectRegistryEntity end diff --git a/ee/spec/finders/geo/project_registry_mismatch_finder_spec.rb b/ee/spec/finders/geo/project_registry_mismatch_finder_spec.rb deleted file mode 100644 index 0cce4337aef..00000000000 --- a/ee/spec/finders/geo/project_registry_mismatch_finder_spec.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Geo::ProjectRegistryMismatchFinder, :geo, :geo_fdw do - describe '#execute' do - let(:node) { create(:geo_node) } - let(:group_1) { create(:group) } - let(:group_2) { create(:group) } - let(:nested_group_1) { create(:group, parent: group_1) } - let(:project_1) { create(:project, group: group_1) } - let(:project_2) { create(:project, group: nested_group_1) } - let(:project_3) { create(:project, group: nested_group_1) } - let(:project_4) { create(:project, :broken_storage, group: group_2) } - let(:project_5) { create(:project, :broken_storage, group: group_2) } - let!(:registry_mismatch) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_checksum_mismatch, project: project_1) } - let!(:registry_repository_mismatch) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_verified, project: project_2) } - let!(:registry_wiki_mismatch) { create(:geo_project_registry, :repository_verified, :wiki_checksum_mismatch, project: project_3) } - let!(:registry_wiki_mismatch_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_checksum_mismatch, project: project_4) } - let!(:registry_repository_mismatch_broken_shard) { create(:geo_project_registry, :repository_checksum_mismatch, :wiki_verified, project: project_5) } - let!(:registry_verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) } - - context 'with repository type' do - subject { described_class.new(current_node: node, type: :repository) } - - context 'without selective sync' do - it 'returns all mismatch registries' do - expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_repository_mismatch_broken_shard) - end - end - - context 'with selective sync by namespace' do - it 'returns mismatch registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch) - end - end - - context 'with selective sync by shard' do - it 'returns mismatch registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to contain_exactly(registry_repository_mismatch_broken_shard) - end - end - end - - context 'with wiki type' do - subject { described_class.new(current_node: node, type: :wiki) } - - context 'without selective sync' do - it 'returns all mismatch registries' do - expect(subject.execute).to contain_exactly(registry_mismatch, registry_wiki_mismatch, registry_wiki_mismatch_broken_shard) - end - end - - context 'with selective sync by namespace' do - it 'returns mismatch registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to contain_exactly(registry_mismatch, registry_wiki_mismatch) - end - end - - context 'with selective sync by shard' do - it 'returns mismatch registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to contain_exactly(registry_wiki_mismatch_broken_shard) - end - end - end - - context 'with invalid type' do - subject { described_class.new(current_node: node, type: :invalid) } - - context 'without selective sync' do - it 'returns all mismatch registries' do - expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_wiki_mismatch, registry_repository_mismatch_broken_shard, registry_wiki_mismatch_broken_shard) - end - end - - context 'with selective sync by namespace' do - it 'returns all mismatch registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to contain_exactly(registry_mismatch, registry_repository_mismatch, registry_wiki_mismatch) - end - end - - context 'with selective sync by shard' do - it 'returns all mismatch registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to contain_exactly(registry_repository_mismatch_broken_shard, registry_wiki_mismatch_broken_shard) - end - end - end - end -end diff --git a/ee/spec/finders/geo/project_registry_sync_failed_finder_spec.rb b/ee/spec/finders/geo/project_registry_sync_failed_finder_spec.rb deleted file mode 100644 index 561a5a40838..00000000000 --- a/ee/spec/finders/geo/project_registry_sync_failed_finder_spec.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Geo::ProjectRegistrySyncFailedFinder, :geo, :geo_fdw do - describe '#execute' do - let(:node) { create(:geo_node) } - let(:group_1) { create(:group) } - let(:group_2) { create(:group) } - let(:nested_group_1) { create(:group, parent: group_1) } - let(:project_1) { create(:project, group: group_1) } - let(:project_2) { create(:project, group: nested_group_1) } - let(:project_3) { create(:project, group: nested_group_1) } - let(:project_4) { create(:project, :broken_storage, group: group_2) } - let(:project_5) { create(:project, :broken_storage, group: group_2) } - let!(:registry_failed) { create(:geo_project_registry, :sync_failed, project: project_1) } - let!(:registry_repository_failed) { create(:geo_project_registry, :synced, :repository_sync_failed, project: project_2) } - let!(:registry_wiki_failed) { create(:geo_project_registry, :synced, :wiki_sync_failed, project: project_3) } - let!(:registry_wiki_failed_broken_shard) { create(:geo_project_registry, :synced, :wiki_sync_failed, project: project_4) } - let!(:registry_repository_failed_broken_shard) { create(:geo_project_registry, :synced, :repository_sync_failed, project: project_5) } - let!(:registry_synced) { create(:geo_project_registry, :synced) } - - context 'with repository type' do - subject { described_class.new(current_node: node, type: :repository) } - - context 'without selective sync' do - it 'returns all failed registries' do - expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_repository_failed_broken_shard]) - end - end - - context 'with selective sync by namespace' do - it 'returns failed registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to match_array([registry_failed, registry_repository_failed]) - end - end - - context 'with selective sync by shard' do - it 'returns failed registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to match_array([registry_repository_failed_broken_shard]) - end - end - end - - context 'with wiki type' do - subject { described_class.new(current_node: node, type: :wiki) } - - context 'without selective sync' do - it 'returns all failed registries' do - expect(subject.execute).to match_array([registry_failed, registry_wiki_failed, registry_wiki_failed_broken_shard]) - end - end - - context 'with selective sync by namespace' do - it 'returns failed registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to match_array([registry_failed, registry_wiki_failed]) - end - end - - context 'with selective sync by shard' do - it 'returns failed registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to match_array([registry_wiki_failed_broken_shard]) - end - end - end - - context 'with no type' do - subject { described_class.new(current_node: node, type: :invalid) } - - context 'without selective sync' do - it 'returns all failed registries' do - expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_wiki_failed, registry_repository_failed_broken_shard, registry_wiki_failed_broken_shard]) - end - end - - context 'with selective sync by namespace' do - it 'returns all failed registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to match_array([registry_failed, registry_repository_failed, registry_wiki_failed]) - end - end - - context 'with selective sync by shard' do - it 'returns all failed registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to match_array([registry_repository_failed_broken_shard, registry_wiki_failed_broken_shard]) - end - end - end - end -end diff --git a/ee/spec/finders/geo/project_registry_verification_failed_finder_spec.rb b/ee/spec/finders/geo/project_registry_verification_failed_finder_spec.rb deleted file mode 100644 index 624a004accd..00000000000 --- a/ee/spec/finders/geo/project_registry_verification_failed_finder_spec.rb +++ /dev/null @@ -1,101 +0,0 @@ -# frozen_string_literal: true - -require 'spec_helper' - -RSpec.describe Geo::ProjectRegistryVerificationFailedFinder, :geo, :geo_fdw do - describe '#execute' do - let(:node) { create(:geo_node) } - let(:group_1) { create(:group) } - let(:group_2) { create(:group) } - let(:nested_group_1) { create(:group, parent: group_1) } - let(:project_1) { create(:project, group: group_1) } - let(:project_2) { create(:project, group: nested_group_1) } - let(:project_3) { create(:project, group: nested_group_1) } - let(:project_4) { create(:project, :broken_storage, group: group_2) } - let(:project_5) { create(:project, :broken_storage, group: group_2) } - let!(:registry_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verification_failed, project: project_1) } - let!(:registry_repository_verification_failed) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_2) } - let!(:registry_wiki_verification_failed) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_3) } - let!(:registry_wiki_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verified, :wiki_verification_failed, project: project_4) } - let!(:registry_repository_verification_failed_broken_shard) { create(:geo_project_registry, :repository_verification_failed, :wiki_verified, project: project_5) } - let!(:registry_verified) { create(:geo_project_registry, :repository_verified, :wiki_verified) } - - context 'with repository type' do - subject { described_class.new(current_node: node, type: :repository) } - - context 'without selective sync' do - it 'returns all failed registries' do - expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_repository_verification_failed_broken_shard) - end - end - - context 'with selective sync by namespace' do - it 'returns failed registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed) - end - end - - context 'with selective sync by shard' do - it 'returns failed registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to contain_exactly(registry_repository_verification_failed_broken_shard) - end - end - end - - context 'with wiki type' do - subject { described_class.new(current_node: node, type: :wiki) } - - context 'without selective sync' do - it 'returns all failed registries' do - expect(subject.execute).to contain_exactly(registry_verification_failed, registry_wiki_verification_failed, registry_wiki_verification_failed_broken_shard) - end - end - - context 'with selective sync by namespace' do - it 'returns failed registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to contain_exactly(registry_verification_failed, registry_wiki_verification_failed) - end - end - - context 'with selective sync by shard' do - it 'returns failed registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to contain_exactly(registry_wiki_verification_failed_broken_shard) - end - end - end - - context 'with no type' do - subject { described_class.new(current_node: node, type: :invalid) } - - context 'without selective sync' do - it 'returns all failed registries' do - expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_wiki_verification_failed, registry_repository_verification_failed_broken_shard, registry_wiki_verification_failed_broken_shard) - end - end - - context 'with selective sync by namespace' do - it 'returns all failed registries where projects belongs to the namespaces' do - node.update!(selective_sync_type: 'namespaces', namespaces: [group_1]) - - expect(subject.execute).to contain_exactly(registry_verification_failed, registry_repository_verification_failed, registry_wiki_verification_failed) - end - end - - context 'with selective sync by shard' do - it 'returns all failed registries where projects belongs to the shards' do - node.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken']) - - expect(subject.execute).to contain_exactly(registry_repository_verification_failed_broken_shard, registry_wiki_verification_failed_broken_shard) - end - end - end - end -end -- 2.30.9