Merge branch '5594-geo-add-repository-verification-failures-to-api' into 'master'

Resolve "Geo: Add repository verification failures to API"

Closes #5594

See merge request gitlab-org/gitlab-ee!6137

doc/api/geo_nodes.md

@@ -338,8 +338,9 @@ Example response:
 
 Please note that the `health_status` parameter can only be in an "Healthy" or "Unhealthy" state, while the `health` parameter can be empty, "Healthy", or contain the actual error message.
 
-## Retrieve project sync failures that occurred on the current node
+## Retrieve project sync or verification failures that occurred on the current node
 
+This only works on a secondary node.
 
 ```
 GET /geo_nodes/current/failures
@@ -347,7 +348,8 @@ GET /geo_nodes/current/failures
 
 | Attribute | Type | Required | Description |
 | --------- | ---- | -------- | ----------- |
-| `type` | string | no | Type of failure (`repository`/`wiki`) |
+| `type`         | string  | no | Type of failed objects (`repository`/`wiki`) |
+| `failure_type` | string | no | Type of failures (`sync`/`checksum_mismatch`/`verification`) |
 
 This endpoint uses [Pagination](README.md#pagination).
 
@@ -368,7 +370,13 @@ Example response:
     "repository_retry_count": null,
     "wiki_retry_count": 1,
     "last_repository_sync_failure": null,
-    "last_wiki_sync_failure": "Error syncing Wiki repository"
+    "last_wiki_sync_failure": "Error syncing Wiki repository",
+    "last_repository_verification_failure": "",
+    "last_wiki_verification_failure": "",
+    "repository_verification_checksum_sha": "da39a3ee5e6b4b0d32e5bfef9a601890afd80709",
+    "wiki_verification_checksum_sha": "da39a3ee5e6b4b0d3255bfef9ef0189aafd80709",
+    "repository_checksum_mismatch": false,
+    "wiki_checksum_mismatch": false
   }
 ]
 ```

ee/app/finders/geo/project_registry_finder.rb

@@ -92,6 +92,14 @@ module Geo
       end
     end
 
+    def find_checksum_mismatch_project_registries(type = nil)
+      if use_legacy_queries?
+        legacy_find_filtered_checksum_mismatch_projects(type)
+      else
+        find_filtered_checksum_mismatch_project_registries(type)
+      end
+    end
+
     # Find all registries that need a repository or wiki verification
     def find_registries_to_verify(batch_size:)
       if use_legacy_queries?
@@ -155,6 +163,17 @@ module Geo
       end
     end
 
+    def find_filtered_checksum_mismatch_project_registries(type = nil)
+      case type
+      when 'repository'
+        Geo::ProjectRegistry.repository_checksum_mismatch
+      when 'wiki'
+        Geo::ProjectRegistry.wiki_checksum_mismatch
+      else
+        Geo::ProjectRegistry.checksum_mismatch
+      end
+    end
+
     #
     # FDW accessors
     #
@@ -292,7 +311,7 @@ module Geo
       )
     end
 
-    # @return [ActiveRecord::Relation<Project>] list of projects that sync has failed
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of projects that sync has failed
     def legacy_find_filtered_failed_projects(type = nil)
       legacy_inner_join_registry_ids(
         find_filtered_failed_project_registries(type),
@@ -302,7 +321,7 @@ module Geo
       )
     end
 
-    # @return [ActiveRecord::Relation<Project>] list of projects that verification has failed
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of projects that verification has failed
     def legacy_find_filtered_verification_failed_projects(type = nil)
       legacy_inner_join_registry_ids(
         find_filtered_verification_failed_project_registries(type),
@@ -312,6 +331,16 @@ module Geo
       )
     end
 
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of projects where there is a checksum_mismatch
+    def legacy_find_filtered_checksum_mismatch_projects(type = nil)
+      legacy_inner_join_registry_ids(
+        find_filtered_checksum_mismatch_project_registries(type),
+        current_node.projects.pluck(:id),
+        Geo::ProjectRegistry,
+        foreign_key: :project_id
+      )
+    end
+
     # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of registries that need verification
     def legacy_find_registries_to_verify(batch_size:)
       repo_condition = local_repo_condition

ee/app/models/geo/project_registry.rb

@@ -46,6 +46,12 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     where(repository_verification_failed.or(wiki_verification_failed))
   end
 
+  def self.checksum_mismatch
+    repository_checksum_mismatch = arel_table[:repository_checksum_mismatch].eq(true)
+    wiki_checksum_mismatch = arel_table[:wiki_checksum_mismatch].eq(true)
+    where(repository_checksum_mismatch.or(wiki_checksum_mismatch))
+  end
+
   def self.retry_due
     where(
       arel_table[:repository_retry_at].lt(Time.now)

ee/app/serializers/geo_project_registry_entity.rb

@@ -8,4 +8,10 @@ class GeoProjectRegistryEntity < Grape::Entity
   expose :wiki_retry_count
   expose :last_repository_sync_failure
   expose :last_wiki_sync_failure
+  expose :last_repository_verification_failure
+  expose :last_wiki_verification_failure
+  expose :repository_verification_checksum_sha
+  expose :wiki_verification_checksum_sha
+  expose :repository_checksum_mismatch
+  expose :wiki_checksum_mismatch
 end

ee/changelogs/unreleased/5594-geo-add-repository-verification-failures-to-api.yml

+---
+title: 'Geo: Add repository verification failures to API'
+merge_request: 6137
+author:
+type: added

ee/lib/api/geo_nodes.rb

@@ -43,17 +43,30 @@ module API
       end
       params do
         optional :type, type: String, values: %w[wiki repository], desc: 'Type of failure (repository/wiki)'
+        optional :failure_type, type: String, default: 'sync', desc: 'Show verification failures'
         use :pagination
       end
       get '/current/failures' do
         geo_node = Gitlab::Geo.current_node
 
+        forbidden!('Failures can only be requested from a secondary node') unless geo_node.secondary?
+
         not_found!('Geo node not found') unless geo_node
 
         finder = ::Geo::ProjectRegistryFinder.new(current_node: geo_node)
-        project_registries = paginate(finder.find_failed_project_registries(params[:type]))
 
-        present project_registries, with: ::GeoProjectRegistryEntity
+        project_registries = case params[:failure_type]
+                             when 'sync'
+                               finder.find_failed_project_registries(params[:type])
+                             when 'verification'
+                               finder.find_verification_failed_project_registries(params[:type])
+                             when 'checksum_mismatch'
+                               finder.find_checksum_mismatch_project_registries(params[:type])
+                             else
+                               not_found!('Failure type unknown')
+                             end
+
+        present paginate(project_registries), with: ::GeoProjectRegistryEntity
       end
 
       route_param :id, type: Integer, desc: 'The ID of the node' do

ee/lib/ee/api/projects.rb

+module EE
+  module API
+    module Projects
+      extend ActiveSupport::Concern
+
+      prepended do
+        helpers do
+          params :optional_filter_params_ee do
+            optional :wiki_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where wiki checksum is failed'
+            optional :repository_checksum_failed, type: Grape::API::Boolean, default: false, desc: 'Limit by projects where repository checksum is failed'
+          end
+
+          def apply_filters(projects)
+            projects = super(projects)
+            projects = projects.verification_failed_wikis if params[:wiki_checksum_failed]
+            projects = projects.verification_failed_repos if params[:repository_checksum_failed]
+
+            projects
+          end
+        end
+      end
+    end
+  end
+end

ee/spec/finders/geo/project_registry_finder_spec.rb

@@ -333,6 +333,62 @@ describe Geo::ProjectRegistryFinder, :geo do
     end
   end
 
+  describe '#find_checksum_mismatch_project_registries' do
+    it 'delegates to #find_filtered_checksum_mismatch_project_registries' do
+      expect(subject).to receive(:find_filtered_checksum_mismatch_project_registries).and_call_original
+
+      subject.find_checksum_mismatch_project_registries
+    end
+
+    it 'delegates to #legacy_find_filtered_checksum_mismatch_projects when use_legacy_queries is true' do
+      expect(subject).to receive(:use_legacy_queries?).and_return(true)
+
+      expect(subject).to receive(:legacy_find_filtered_checksum_mismatch_projects).and_call_original
+
+      subject.find_checksum_mismatch_project_registries
+    end
+
+    it 'delegates to #find_filtered_checksum_mismatch_project_registries when use_legacy_queries is false' do
+      expect(subject).to receive(:use_legacy_queries?).and_return(false)
+
+      expect(subject).to receive(:find_filtered_checksum_mismatch_project_registries).and_call_original
+
+      subject.find_checksum_mismatch_project_registries
+    end
+
+    it 'counts projects with a checksum mismatch' do
+      repository_mismatch1 = create(:geo_project_registry, :repository_checksum_mismatch)
+      repository_mismatch2 = create(:geo_project_registry, :repository_checksum_mismatch)
+      create(:geo_project_registry, :wiki_verified)
+      wiki_mismatch = create(:geo_project_registry, :wiki_checksum_mismatch)
+
+      expect(subject.find_checksum_mismatch_project_registries('wiki')).to match_array(wiki_mismatch)
+      expect(subject.find_checksum_mismatch_project_registries('repository')).to match_array([repository_mismatch1, repository_mismatch2])
+      expect(subject.find_checksum_mismatch_project_registries(nil)).to match_array([repository_mismatch1, repository_mismatch2, wiki_mismatch])
+    end
+
+    context 'with selective sync' do
+      before do
+        secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
+      end
+
+      it 'delegates to #legacy_find_filtered_checksum_mismatch_projects' do
+        expect(subject).to receive(:legacy_find_filtered_checksum_mismatch_projects).and_call_original
+
+        subject.find_checksum_mismatch_project_registries
+      end
+
+      it 'returns projects with a checksum mismatch' do
+        project_1_in_synced_group = create(:project, group: synced_group)
+        project_1_registry_record = create(:geo_project_registry, :repository_checksum_mismatch, project: project_1_in_synced_group)
+
+        projects = subject.find_checksum_mismatch_project_registries('repository')
+
+        expect(projects).to match_ids(project_1_registry_record)
+      end
+    end
+  end
+
   shared_examples 'finds all the things' do
     describe '#find_unsynced_projects' do
       it 'delegates to the correct method' do

ee/spec/models/geo/project_registry_spec.rb

@@ -121,6 +121,16 @@ describe Geo::ProjectRegistry do
     end
   end
 
+  describe '.checksum_mismatch' do
+    it 'returns projects where there is a checksum mismatch' do
+      registry_repository_checksum_mismatch = create(:geo_project_registry, :repository_checksum_mismatch)
+      regisry_wiki_checksum_mismatch = create(:geo_project_registry, :wiki_checksum_mismatch)
+      create(:geo_project_registry)
+
+      expect(described_class.checksum_mismatch).to match_array([regisry_wiki_checksum_mismatch, registry_repository_checksum_mismatch])
+    end
+  end
+
   describe '.retry_due' do
     it 'returns projects that should be synced' do
       create(:geo_project_registry, repository_retry_at: Date.yesterday, wiki_retry_at: Date.yesterday)

ee/spec/requests/api/geo_nodes_spec.rb

@@ -215,78 +215,186 @@ describe API::GeoNodes, :geo, :prometheus, api: true do
     end
   end
 
-  describe 'GET /geo_nodes/current/failures/:type' do
-    it 'fetches the current node failures' do
-      create(:geo_project_registry, :sync_failed)
-      create(:geo_project_registry, :sync_failed)
-
-      stub_current_geo_node(secondary)
-      expect(Gitlab::Geo).to receive(:current_node).and_return(secondary)
+  describe 'GET /geo_nodes/current/failures' do
+    context 'primary node' do
+      before do
+        stub_current_geo_node(primary)
+      end
 
-      get api("/geo_nodes/current/failures", admin)
+      it 'forbids requests' do
+        get api("/geo_nodes/current/failures", admin)
 
-      expect(response).to have_gitlab_http_status(200)
-      expect(response).to match_response_schema('public_api/v4/geo_project_registry', dir: 'ee')
+        expect(response).to have_gitlab_http_status(403)
+      end
     end
 
-    it 'does not show any registry when there is no failure' do
-      create(:geo_project_registry, :synced)
-
-      stub_current_geo_node(secondary)
-      expect(Gitlab::Geo).to receive(:current_node).and_return(secondary)
+    context 'secondary node' do
+      before do
+        stub_current_geo_node(secondary)
+      end
 
-      get api("/geo_nodes/current/failures", admin)
+      it 'fetches the current node failures' do
+        create(:geo_project_registry, :sync_failed)
+        create(:geo_project_registry, :sync_failed)
 
-      expect(response).to have_gitlab_http_status(200)
-      expect(json_response.count).to be_zero
-    end
+        get api("/geo_nodes/current/failures", admin)
 
-    context 'wiki type' do
-      it 'only shows wiki failures' do
-        create(:geo_project_registry, :wiki_sync_failed)
-        create(:geo_project_registry, :repository_sync_failed)
+        expect(response).to have_gitlab_http_status(200)
+        expect(response).to match_response_schema('public_api/v4/geo_project_registry', dir: 'ee')
+      end
 
-        stub_current_geo_node(secondary)
-        expect(Gitlab::Geo).to receive(:current_node).and_return(secondary)
+      it 'does not show any registry when there is no failure' do
+        create(:geo_project_registry, :synced)
 
-        get api("/geo_nodes/current/failures?type=wiki", admin)
+        get api("/geo_nodes/current/failures", admin)
 
         expect(response).to have_gitlab_http_status(200)
-        expect(json_response.count).to eq(1)
-        expect(json_response.first['wiki_retry_count']).to be > 0
+        expect(json_response.count).to be_zero
       end
-    end
 
-    context 'repository type' do
-      it 'only shows repository failures' do
-        create(:geo_project_registry, :wiki_sync_failed)
-        create(:geo_project_registry, :repository_sync_failed)
+      context 'wiki type' do
+        it 'only shows wiki failures' do
+          create(:geo_project_registry, :wiki_sync_failed)
+          create(:geo_project_registry, :repository_sync_failed)
 
-        stub_current_geo_node(secondary)
-        expect(Gitlab::Geo).to receive(:current_node).and_return(secondary)
+          get api("/geo_nodes/current/failures", admin), type: :wiki
 
-        get api("/geo_nodes/current/failures?type=repository", admin)
+          expect(response).to have_gitlab_http_status(200)
+          expect(json_response.count).to eq(1)
+          expect(json_response.first['wiki_retry_count']).to be > 0
+        end
+      end
 
-        expect(response).to have_gitlab_http_status(200)
-        expect(json_response.count).to eq(1)
-        expect(json_response.first['repository_retry_count']).to be > 0
+      context 'repository type' do
+        it 'only shows repository failures' do
+          create(:geo_project_registry, :wiki_sync_failed)
+          create(:geo_project_registry, :repository_sync_failed)
+
+          get api("/geo_nodes/current/failures", admin), type: :repository
+
+          expect(response).to have_gitlab_http_status(200)
+          expect(json_response.count).to eq(1)
+          expect(json_response.first['repository_retry_count']).to be > 0
+        end
       end
-    end
 
-    context 'nonexistent type' do
-      it 'returns a bad request' do
-        create(:geo_project_registry, :repository_sync_failed)
+      context 'nonexistent type' do
+        it 'returns a bad request' do
+          create(:geo_project_registry, :repository_sync_failed)
 
-        get api("/geo_nodes/current/failures?type=nonexistent", admin)
+          get api("/geo_nodes/current/failures", admin), type: :nonexistent
 
-        expect(response).to have_gitlab_http_status(400)
+          expect(response).to have_gitlab_http_status(400)
+        end
       end
-    end
 
-    it 'denies access if not admin' do
-      get api("/geo_nodes/current/failures", user)
+      it 'denies access if not admin' do
+        get api("/geo_nodes/current/failures", user)
 
-      expect(response).to have_gitlab_http_status(403)
+        expect(response).to have_gitlab_http_status(403)
+      end
+
+      context 'verification failures' do
+        before do
+          stub_current_geo_node(secondary)
+        end
+
+        it 'fetches the current node checksum failures' do
+          create(:geo_project_registry, :repository_verification_failed)
+          create(:geo_project_registry, :wiki_verification_failed)
+
+          get api("/geo_nodes/current/failures", admin), failure_type: 'verification'
+
+          expect(response).to have_gitlab_http_status(200)
+          expect(response).to match_response_schema('public_api/v4/geo_project_registry', dir: 'ee')
+        end
+
+        it 'does not show any registry when there is no failure' do
+          create(:geo_project_registry, :repository_verified)
+
+          get api("/geo_nodes/current/failures", admin), failure_type: 'verification'
+
+          expect(response).to have_gitlab_http_status(200)
+          expect(json_response.count).to be_zero
+        end
+
+        context 'wiki type' do
+          it 'only shows wiki verification failures' do
+            create(:geo_project_registry, :repository_verification_failed)
+            create(:geo_project_registry, :wiki_verification_failed)
+
+            get api("/geo_nodes/current/failures", admin), failure_type: 'verification', type: :wiki
+
+            expect(response).to have_gitlab_http_status(200)
+            expect(json_response.count).to eq(1)
+            expect(json_response.first['last_wiki_verification_failure']).to be_present
+          end
+        end
+
+        context 'repository type' do
+          it 'only shows repository failures' do
+            create(:geo_project_registry, :repository_verification_failed)
+            create(:geo_project_registry, :wiki_verification_failed)
+
+            get api("/geo_nodes/current/failures", admin), failure_type: 'verification', type: :repository
+
+            expect(response).to have_gitlab_http_status(200)
+            expect(json_response.count).to eq(1)
+            expect(json_response.first['last_repository_verification_failure']).to be_present
+          end
+        end
+      end
+
+      context 'checksum mismatch failures' do
+        before do
+          stub_current_geo_node(secondary)
+        end
+
+        it 'fetches the checksum mismatch failures from current node' do
+          create(:geo_project_registry, :repository_checksum_mismatch)
+          create(:geo_project_registry, :wiki_checksum_mismatch)
+
+          get api("/geo_nodes/current/failures", admin), failure_type: 'checksum_mismatch'
+
+          expect(response).to have_gitlab_http_status(200)
+          expect(response).to match_response_schema('public_api/v4/geo_project_registry', dir: 'ee')
+        end
+
+        it 'does not show any registry when there is no failure' do
+          create(:geo_project_registry, :repository_verified)
+
+          get api("/geo_nodes/current/failures", admin), failure_type: 'checksum_mismatch'
+
+          expect(response).to have_gitlab_http_status(200)
+          expect(json_response.count).to be_zero
+        end
+
+        context 'wiki type' do
+          it 'only shows wiki checksum mismatch failures' do
+            create(:geo_project_registry, :repository_checksum_mismatch)
+            create(:geo_project_registry, :wiki_checksum_mismatch)
+
+            get api("/geo_nodes/current/failures", admin), failure_type: 'checksum_mismatch', type: :wiki
+
+            expect(response).to have_gitlab_http_status(200)
+            expect(json_response.count).to eq(1)
+            expect(json_response.first['wiki_checksum_mismatch']).to be_truthy
+          end
+        end
+
+        context 'repository type' do
+          it 'only shows repository checksum mismatch failures' do
+            create(:geo_project_registry, :repository_checksum_mismatch)
+            create(:geo_project_registry, :wiki_checksum_mismatch)
+
+            get api("/geo_nodes/current/failures", admin), failure_type: 'checksum_mismatch', type: :repository
+
+            expect(response).to have_gitlab_http_status(200)
+            expect(json_response.count).to eq(1)
+            expect(json_response.first['repository_checksum_mismatch']).to be_truthy
+          end
+        end
+      end
     end
   end
 end

ee/spec/requests/api/projects_spec.rb

@@ -19,4 +19,36 @@ describe API::Projects do
       expect(project.reload.external_authorization_classification_label).to eq('new label')
     end
   end
+
+  describe 'GET /projects' do
+    context 'filters by verification flags' do
+      let(:project1) { create(:project, namespace: user.namespace) }
+
+      it 'filters by :repository_verification_failed' do
+        create(:repository_state, :repository_failed, project: project)
+        create(:repository_state, :wiki_failed, project: project1)
+
+        get api('/projects', user), repository_checksum_failed: true
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response).to include_pagination_headers
+        expect(json_response).to be_an Array
+        expect(json_response.length).to eq(1)
+        expect(json_response.first['id']).to eq project.id
+      end
+
+      it 'filters by :wiki_verification_failed' do
+        create(:repository_state, :wiki_failed, project: project)
+        create(:repository_state, :repository_failed, project: project1)
+
+        get api('/projects', user), wiki_checksum_failed: true
+
+        expect(response).to have_gitlab_http_status(200)
+        expect(response).to include_pagination_headers
+        expect(json_response).to be_an Array
+        expect(json_response.length).to eq(1)
+        expect(json_response.first['id']).to eq project.id
+      end
+    end
+  end
 end

lib/api/projects.rb

@@ -23,6 +23,8 @@ module API
       end
     end
 
+    prepend EE::API::Projects
+
     helpers do
       params :statistics_params do
         optional :statistics, type: Boolean, default: false, desc: 'Include project statistics'