Merge branch 'mk/do-not-checksum-remote-files' into 'master'

Geo: Exclude remote stored blobs from checksumming

See merge request gitlab-org/gitlab!54132

ee/app/assets/javascripts/geo_nodes/components/node_detail_sections/node_details_section_verification.vue

@@ -48,7 +48,9 @@ export default {
     },
     itemValue(nodeDetailItem) {
       return {
-        totalCount: nodeDetailItem.itemValue.totalCount,
+        totalCount: this.nodeTypePrimary
+          ? nodeDetailItem.itemValue.checksumTotalCount
+          : nodeDetailItem.itemValue.verificationTotalCount,
         successCount: this.nodeTypePrimary
           ? nodeDetailItem.itemValue.checksumSuccessCount
           : nodeDetailItem.itemValue.verificationSuccessCount,

ee/app/assets/javascripts/geo_nodes/store/geo_nodes_store.js

@@ -76,9 +76,12 @@ export default class GeoNodesStore {
           totalCount: rawNodeDetails[`${replicable.namePlural}_count`],
           successCount: rawNodeDetails[`${replicable.namePlural}_synced_count`],
           failureCount: rawNodeDetails[`${replicable.namePlural}_failed_count`],
+          verificationTotalCount:
+            rawNodeDetails[`${replicable.namePlural}_verification_total_count`],
           verificationSuccessCount: rawNodeDetails[`${replicable.namePlural}_verified_count`],
           verificationFailureCount:
             rawNodeDetails[`${replicable.namePlural}_verification_failed_count`],
+          checksumTotalCount: rawNodeDetails[`${replicable.namePlural}_checksum_total_count`],
           checksumSuccessCount: rawNodeDetails[`${replicable.namePlural}_checksummed_count`],
           checksumFailureCount: rawNodeDetails[`${replicable.namePlural}_checksum_failed_count`],
         },

ee/app/models/concerns/geo/verifiable_registry.rb

@@ -36,6 +36,7 @@ module Geo::VerifiableRegistry
     extend ::Gitlab::Utils::Override
 
     sha_attribute :verification_checksum_mismatched
+    scope :available_verifiables, -> { all }
 
     override :clear_verification_failure_fields!
     def clear_verification_failure_fields!

ee/app/models/concerns/geo/verifiable_replicator.rb

@@ -128,7 +128,7 @@ module Geo
         # Bonus: This causes the progress bar to be hidden.
         return unless verification_enabled?
 
-        model.available_replicables.verification_succeeded.count
+        model.verification_succeeded.count
       end
 
       def checksum_failed_count
@@ -136,7 +136,15 @@ module Geo
         # Bonus: This causes the progress bar to be hidden.
         return unless verification_enabled?
 
-        model.available_replicables.verification_failed.count
+        model.verification_failed.count
+      end
+
+      def checksum_total_count
+        # When verification is disabled, this returns nil.
+        # Bonus: This causes the progress bar to be hidden.
+        return unless verification_enabled?
+
+        model.available_verifiables.count
       end
 
       def verified_count
@@ -154,6 +162,14 @@ module Geo
 
         registry_class.synced.verification_failed.count
       end
+
+      def verification_total_count
+        # When verification is disabled, this returns nil.
+        # Bonus: This causes the progress bar to be hidden.
+        return unless verification_enabled?
+
+        registry_class.synced.available_verifiables.count
+      end
     end
 
     def handle_after_checksum_succeeded

ee/app/models/geo_node_status.rb

@@ -34,11 +34,13 @@ class GeoNodeStatus < ApplicationRecord
   def self.status_fields_for(replicable_class)
     {
       "#{replicable_class.replicable_name_plural}_count".to_sym => "Number of #{replicable_class.replicable_title_plural} on the primary",
+      "#{replicable_class.replicable_name_plural}_checksum_total_count".to_sym => "Number of #{replicable_class.replicable_title_plural} available to checksum on primary",
       "#{replicable_class.replicable_name_plural}_checksummed_count".to_sym => "Number of #{replicable_class.replicable_title_plural} checksummed on the primary",
       "#{replicable_class.replicable_name_plural}_checksum_failed_count".to_sym => "Number of #{replicable_class.replicable_title_plural} failed to checksum on primary",
       "#{replicable_class.replicable_name_plural}_synced_count".to_sym => "Number of #{replicable_class.replicable_title_plural} in the registry",
       "#{replicable_class.replicable_name_plural}_failed_count".to_sym => "Number of #{replicable_class.replicable_title_plural} synced on secondary",
       "#{replicable_class.replicable_name_plural}_registry_count".to_sym => "Number of #{replicable_class.replicable_title_plural} failed to sync on secondary",
+      "#{replicable_class.replicable_name_plural}_verification_total_count".to_sym => "Number of #{replicable_class.replicable_title_plural} available to verify on secondary",
       "#{replicable_class.replicable_name_plural}_verified_count".to_sym => "Number of #{replicable_class.replicable_title_plural} verified on the secondary",
       "#{replicable_class.replicable_name_plural}_verification_failed_count".to_sym => "Number of #{replicable_class.replicable_title_plural} failed to verify on secondary"
     }
@@ -72,17 +74,21 @@ class GeoNodeStatus < ApplicationRecord
     job_artifacts_failed_count
     repositories_verified_count
     repositories_verification_failed_count
+    repositories_verification_total_count
     wikis_verified_count
     wikis_verification_failed_count
+    wikis_verification_total_count
     lfs_objects_synced_missing_on_primary_count
     job_artifacts_synced_missing_on_primary_count
     attachments_synced_missing_on_primary_count
     repositories_checksummed_count
     repositories_checksum_failed_count
     repositories_checksum_mismatch_count
+    repositories_checksum_total_count
     wikis_checksummed_count
     wikis_checksum_failed_count
     wikis_checksum_mismatch_count
+    wikis_checksum_total_count
     repositories_retrying_verification_count
     wikis_retrying_verification_count
     projects_count
@@ -562,21 +568,26 @@ class GeoNodeStatus < ApplicationRecord
   def load_primary_verification_data
     self.repositories_checksummed_count = repository_verification_finder.count_verified_repositories
     self.repositories_checksum_failed_count = repository_verification_finder.count_verification_failed_repositories
+    self.repositories_checksum_total_count = self.projects_count
     self.wikis_checksummed_count = repository_verification_finder.count_verified_wikis
     self.wikis_checksum_failed_count = repository_verification_finder.count_verification_failed_wikis
+    self.wikis_checksum_total_count = self.projects_count
 
     Gitlab::Geo::REPLICATOR_CLASSES.each do |replicator|
       public_send("#{replicator.replicable_name_plural}_checksummed_count=", replicator.checksummed_count) # rubocop:disable GitlabSecurity/PublicSend
       public_send("#{replicator.replicable_name_plural}_checksum_failed_count=", replicator.checksum_failed_count) # rubocop:disable GitlabSecurity/PublicSend
+      public_send("#{replicator.replicable_name_plural}_checksum_total_count=", replicator.checksum_total_count) # rubocop:disable GitlabSecurity/PublicSend
     end
   end
 
   def load_secondary_verification_data
     self.repositories_verified_count = Geo::ProjectRegistry.verified(:repository).count
     self.repositories_verification_failed_count = Geo::ProjectRegistry.verification_failed(:repository).count
+    self.repositories_verification_total_count = self.projects_count
     self.repositories_checksum_mismatch_count = Geo::ProjectRegistry.mismatch(:repository).count
     self.wikis_verified_count = Geo::ProjectRegistry.verified(:wiki).count
     self.wikis_verification_failed_count = Geo::ProjectRegistry.verification_failed(:wiki).count
+    self.wikis_verification_total_count = self.projects_count
     self.wikis_checksum_mismatch_count = Geo::ProjectRegistry.mismatch(:wiki).count
     self.repositories_retrying_verification_count = Geo::ProjectRegistry.retrying_verification(:repository).count
     self.wikis_retrying_verification_count = Geo::ProjectRegistry.retrying_verification(:wiki).count
@@ -584,6 +595,7 @@ class GeoNodeStatus < ApplicationRecord
     ::Gitlab::Geo.verification_enabled_replicator_classes.each do |replicator|
       public_send("#{replicator.replicable_name_plural}_verified_count=", replicator.verified_count) # rubocop:disable GitlabSecurity/PublicSend
       public_send("#{replicator.replicable_name_plural}_verification_failed_count=", replicator.verification_failed_count) # rubocop:disable GitlabSecurity/PublicSend
+      public_send("#{replicator.replicable_name_plural}_verification_total_count=", replicator.verification_total_count) # rubocop:disable GitlabSecurity/PublicSend
     end
   end
 

ee/lib/gitlab/geo/replicable_model.rb

@@ -18,7 +18,10 @@ module Gitlab
         # all models. https://gitlab.com/gitlab-org/gitlab/-/issues/280768
         scope :verification_succeeded, -> { none }
         scope :verification_failed, -> { none }
+
+        # These scopes are intended to be overridden as needed
         scope :available_replicables, -> { all }
+        scope :available_verifiables, -> { self.respond_to?(:with_files_stored_locally) ? available_replicables.with_files_stored_locally : available_replicables }
       end
 
       class_methods do

ee/lib/gitlab/geo/verification_state.rb

@@ -28,15 +28,15 @@ module Gitlab
         sha_attribute :verification_checksum
 
         # rubocop:disable CodeReuse/ActiveRecord
-        scope :verification_pending, -> { with_verification_state(:verification_pending) }
-        scope :verification_started, -> { with_verification_state(:verification_started) }
-        scope :verification_succeeded, -> { with_verification_state(:verification_succeeded) }
-        scope :verification_failed, -> { with_verification_state(:verification_failed) }
+        scope :verification_pending, -> { available_verifiables.with_verification_state(:verification_pending) }
+        scope :verification_started, -> { available_verifiables.with_verification_state(:verification_started) }
+        scope :verification_succeeded, -> { available_verifiables.with_verification_state(:verification_succeeded) }
+        scope :verification_failed, -> { available_verifiables.with_verification_state(:verification_failed) }
         scope :checksummed, -> { where.not(verification_checksum: nil) }
         scope :not_checksummed, -> { where(verification_checksum: nil) }
         scope :verification_timed_out, -> { verification_started.where("verification_started_at < ?", VERIFICATION_TIMEOUT.ago) }
         scope :retry_due, -> { where(arel_table[:verification_retry_at].eq(nil).or(arel_table[:verification_retry_at].lt(Time.current))) }
-        scope :needs_verification, -> { with_verification_state(:verification_pending).or(with_verification_state(:verification_failed).retry_due) }
+        scope :needs_verification, -> { available_verifiables.merge(with_verification_state(:verification_pending).or(with_verification_state(:verification_failed).retry_due)) }
         scope :needs_reverification, -> { verification_succeeded.where("verified_at < ?", ::Gitlab::Geo.current_node.minimum_reverification_interval.days.ago) }
         # rubocop:enable CodeReuse/ActiveRecord
 

ee/spec/factories/geo_node_statuses.rb

@@ -35,13 +35,17 @@ FactoryBot.define do
       wikis_failed_count { 1 }
       repositories_checksummed_count { 600 }
       repositories_checksum_failed_count { 120 }
+      repositories_checksum_total_count { 120 }
       wikis_checksummed_count { 585 }
       wikis_checksum_failed_count { 55 }
+      wikis_checksum_total_count { 55 }
       repositories_verified_count { 501 }
       repositories_verification_failed_count { 100 }
+      repositories_verification_total_count { 100 }
       repositories_checksum_mismatch_count { 15 }
       wikis_verified_count { 499 }
       wikis_verification_failed_count { 99 }
+      wikis_verification_total_count { 99 }
       wikis_checksum_mismatch_count { 10 }
       repositories_retrying_verification_count { 25 }
       wikis_retrying_verification_count { 3 }
@@ -80,11 +84,15 @@ FactoryBot.define do
 
       repositories_synced_count { 10 }
       repositories_checksummed_count { 10 }
+      repositories_checksum_total_count { 10 }
       repositories_verified_count { 10 }
+      repositories_verification_total_count { 10 }
       repositories_checked_count { 10 }
       wikis_synced_count { 10 }
       wikis_checksummed_count { 10 }
+      wikis_checksum_total_count { 10 }
       wikis_verified_count { 10 }
+      wikis_verification_total_count { 10 }
       lfs_objects_synced_count { 10 }
       job_artifacts_synced_count { 10 }
       attachments_synced_count { 10 }

ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -42,56 +42,68 @@
     "repository_verification_enabled",
     "repositories_checksummed_count",
     "repositories_checksum_failed_count",
+    "repositories_checksum_total_count",
     "repositories_checksummed_in_percentage",
     "wikis_checksummed_count",
     "wikis_checksum_failed_count",
+    "wikis_checksum_total_count",
     "wikis_checksummed_in_percentage",
     "merge_request_diffs_count",
     "merge_request_diffs_checksum_failed_count",
+    "merge_request_diffs_checksum_total_count",
     "merge_request_diffs_checksummed_count",
     "merge_request_diffs_registry_count",
     "merge_request_diffs_failed_count",
     "merge_request_diffs_synced_count",
     "merge_request_diffs_synced_in_percentage",
     "merge_request_diffs_verification_failed_count",
+    "merge_request_diffs_verification_total_count",
     "merge_request_diffs_verified_count",
     "merge_request_diffs_verified_in_percentage",
     "package_files_count",
     "package_files_checksum_failed_count",
+    "package_files_checksum_total_count",
     "package_files_checksummed_count",
     "package_files_registry_count",
     "package_files_failed_count",
     "package_files_synced_count",
     "package_files_synced_in_percentage",
     "package_files_verification_failed_count",
+    "package_files_verification_total_count",
     "package_files_verified_count",
     "package_files_verified_in_percentage",
     "terraform_state_versions_count",
     "terraform_state_versions_checksum_failed_count",
+    "terraform_state_versions_checksum_total_count",
     "terraform_state_versions_checksummed_count",
     "terraform_state_versions_registry_count",
     "terraform_state_versions_failed_count",
     "terraform_state_versions_synced_count",
     "terraform_state_versions_synced_in_percentage",
     "terraform_state_versions_verification_failed_count",
+    "terraform_state_versions_verification_total_count",
     "terraform_state_versions_verified_count",
     "terraform_state_versions_verified_in_percentage",
     "snippet_repositories_count",
     "snippet_repositories_checksum_failed_count",
+    "snippet_repositories_checksum_total_count",
     "snippet_repositories_checksummed_count",
     "snippet_repositories_registry_count",
     "snippet_repositories_failed_count",
     "snippet_repositories_synced_count",
     "snippet_repositories_synced_in_percentage",
     "snippet_repositories_verification_failed_count",
+    "snippet_repositories_verification_total_count",
     "snippet_repositories_verified_count",
     "snippet_repositories_verified_in_percentage",
     "repositories_verified_count",
     "repositories_verification_failed_count",
+    "repositories_verification_total_count",
     "repositories_verified_in_percentage",
     "repositories_checksum_mismatch_count",
     "wikis_verified_count",
     "wikis_verification_failed_count",
+    "wikis_verification_total_count",
     "wikis_verified_in_percentage",
     "wikis_checksum_mismatch_count",
     "repositories_retrying_verification_count",
@@ -163,56 +175,68 @@
     "wikis_synced_in_percentage": { "type": "string" },
     "repositories_checksummed_count": { "type": ["integer", "null"] },
     "repositories_checksum_failed_count": { "type": ["integer", "null"] },
+    "repositories_checksum_total_count": { "type": ["integer", "null"] },
     "repositories_checksummed_in_percentage": { "type": "string" },
     "wikis_checksummed_count": { "type": ["integer", "null"] },
     "wikis_checksum_failed_count": { "type": ["integer", "null"] },
+    "wikis_checksum_total_count": { "type": ["integer", "null"] },
     "wikis_checksummed_in_percentage": { "type": "string" },
     "merge_request_diffs_count": { "type": ["integer", "null"] },
     "merge_request_diffs_checksummed_count": { "type": ["integer", "null"] },
     "merge_request_diffs_checksum_failed_count": { "type": ["integer", "null"] },
+    "merge_request_diffs_checksum_total_count": { "type": ["integer", "null"] },
     "merge_request_diffs_registry_count": { "type": ["integer", "null"] },
     "merge_request_diffs_failed_count": { "type": ["integer", "null"] },
     "merge_request_diffs_synced_count": { "type": ["integer", "null"] },
     "merge_request_diffs_synced_in_percentage": { "type": "string" },
     "merge_request_diffs_verification_failed_count": { "type": ["integer", "null"] },
+    "merge_request_diffs_verification_total_count": { "type": ["integer", "null"] },
     "merge_request_diffs_verified_count": { "type": ["integer", "null"] },
     "merge_request_diffs_verified_in_percentage": { "type": "string" },
     "package_files_count": { "type": ["integer", "null"] },
     "package_files_checksummed_count": { "type": ["integer", "null"] },
     "package_files_checksum_failed_count": { "type": ["integer", "null"] },
+    "package_files_checksum_total_count": { "type": ["integer", "null"] },
     "package_files_registry_count": { "type": ["integer", "null"] },
     "package_files_failed_count": { "type": ["integer", "null"] },
     "package_files_synced_count": { "type": ["integer", "null"] },
     "package_files_synced_in_percentage": { "type": "string" },
     "package_files_verification_failed_count": { "type": ["integer", "null"] },
+    "package_files_verification_total_count": { "type": ["integer", "null"] },
     "package_files_verified_count": { "type": ["integer", "null"] },
     "package_files_verified_in_percentage": { "type": "string" },
     "terraform_state_versions_count": { "type": ["integer", "null"] },
     "terraform_state_versions_checksummed_count": { "type": ["integer", "null"] },
     "terraform_state_versions_checksum_failed_count": { "type": ["integer", "null"] },
+    "terraform_state_versions_checksum_total_count": { "type": ["integer", "null"] },
     "terraform_state_versions_registry_count": { "type": ["integer", "null"] },
     "terraform_state_versions_failed_count": { "type": ["integer", "null"] },
     "terraform_state_versions_synced_count": { "type": ["integer", "null"] },
     "terraform_state_versions_synced_in_percentage": { "type": "string" },
     "terraform_state_versions_verification_failed_count": { "type": ["integer", "null"] },
+    "terraform_state_versions_verification_total_count": { "type": ["integer", "null"] },
     "terraform_state_versions_verified_count": { "type": ["integer", "null"] },
     "terraform_state_versions_verified_in_percentage": { "type": "string" },
     "snippet_repositories_count": { "type": ["integer", "null"] },
     "snippet_repositories_checksummed_count": { "type": ["integer", "null"] },
     "snippet_repositories_checksum_failed_count": { "type": ["integer", "null"] },
+    "snippet_repositories_checksum_total_count": { "type": ["integer", "null"] },
     "snippet_repositories_registry_count": { "type": ["integer", "null"] },
     "snippet_repositories_failed_count": { "type": ["integer", "null"] },
     "snippet_repositories_synced_count": { "type": ["integer", "null"] },
     "snippet_repositories_synced_in_percentage": { "type": "string" },
     "snippet_repositories_verification_failed_count": { "type": ["integer", "null"] },
+    "snippet_repositories_verification_total_count": { "type": ["integer", "null"] },
     "snippet_repositories_verified_count": { "type": ["integer", "null"] },
     "snippet_repositories_verified_in_percentage": { "type": "string" },
     "repositories_verified_count": { "type": ["integer", "null"] },
     "repositories_verification_failed_count": { "type": ["integer", "null"] },
+    "repositories_verification_total_count": { "type": ["integer", "null"] },
     "repositories_verified_in_percentage": { "type": "string" },
     "repositories_checksum_mismatch_count": { "type": ["integer", "null"] },
     "wikis_verified_count": { "type": ["integer", "null"] },
     "wikis_verification_failed_count": { "type": ["integer", "null"] },
+    "wikis_verification_total_count": { "type": ["integer", "null"] },
     "wikis_verified_in_percentage": { "type": "string" },
     "wikis_checksum_mismatch_count": { "type": ["integer", "null"] },
     "repositories_retrying_verification_count": { "type": ["integer", "null"] },

ee/spec/support/shared_examples/models/concerns/verifiable_replicator_shared_examples.rb

@@ -58,14 +58,14 @@ RSpec.shared_examples 'a verifiable replicator' do
         allow(described_class).to receive(:verification_enabled?).and_return(true)
       end
 
-      it 'returns the number of available replicables where verification succeeded' do
-        model_record.verification_started!
+      it 'returns the number of available verifiables where verification succeeded' do
+        model_record.verification_started
         model_record.verification_succeeded_with_checksum!('some checksum', Time.current)
 
         expect(described_class.checksummed_count).to eq(1)
       end
 
-      it 'excludes other verification states' do
+      it 'excludes non-success verification states' do
         model_record.verification_started!
 
         expect(described_class.checksummed_count).to eq(0)
@@ -95,7 +95,7 @@ RSpec.shared_examples 'a verifiable replicator' do
         allow(described_class).to receive(:verification_enabled?).and_return(true)
       end
 
-      it 'returns the number of available replicables where verification failed' do
+      it 'returns the number of available verifiables where verification failed' do
         model_record.verification_started!
         model_record.verification_failed_with_message!('some error message')