Geo: Verification of Uploads

.gitlab/issue_templates/Geo Replicate a new Git repository type.md

@@ -491,6 +491,9 @@ That's all of the required database changes.
       self.primary_key = :cool_widget_id
 
       belongs_to :cool_widget, inverse_of: :cool_widget_state
+
+      validates :verification_failure, length: { maximum: 255 }
+      validates :verification_state, :cool_widget, presence: true
     end
   end
   ```

.gitlab/issue_templates/Geo Replicate a new blob type.md

@@ -455,6 +455,9 @@ That's all of the required database changes.
       self.primary_key = :cool_widget_id
 
       belongs_to :cool_widget, inverse_of: :cool_widget_state
+
+      validates :verification_failure, length: { maximum: 255 }
+      validates :verification_state, :cool_widget, presence: true
     end
   end
   ```

db/migrate/20211101165656_create_upload_states.rb

+# frozen_string_literal: true
+
+class CreateUploadStates < Gitlab::Database::Migration[1.0]
+  VERIFICATION_STATE_INDEX_NAME = "index_upload_states_on_verification_state"
+  PENDING_VERIFICATION_INDEX_NAME = "index_upload_states_pending_verification"
+  FAILED_VERIFICATION_INDEX_NAME = "index_upload_states_failed_verification"
+  NEEDS_VERIFICATION_INDEX_NAME = "index_upload_states_needs_verification"
+
+  disable_ddl_transaction!
+
+  def up
+    create_table :upload_states, id: false do |t|
+      t.datetime_with_timezone :verification_started_at
+      t.datetime_with_timezone :verification_retry_at
+      t.datetime_with_timezone :verified_at
+      t.references :upload, primary_key: true, null: false, foreign_key: { on_delete: :cascade }
+      t.integer :verification_state, default: 0, limit: 2, null: false
+      t.integer :verification_retry_count, limit: 2
+      t.binary :verification_checksum, using: 'verification_checksum::bytea'
+      t.text :verification_failure, limit: 255
+
+      t.index :verification_state, name: VERIFICATION_STATE_INDEX_NAME
+      t.index :verified_at, where: "(verification_state = 0)", order: { verified_at: 'ASC NULLS FIRST' }, name: PENDING_VERIFICATION_INDEX_NAME
+      t.index :verification_retry_at, where: "(verification_state = 3)", order: { verification_retry_at: 'ASC NULLS FIRST' }, name: FAILED_VERIFICATION_INDEX_NAME
+      t.index :verification_state, where: "(verification_state = 0 OR verification_state = 3)", name: NEEDS_VERIFICATION_INDEX_NAME
+    end
+  end
+
+  def down
+    drop_table :upload_states
+  end
+end

db/schema_migrations/20211101165656

+853e68aa974f49b7ab9f60acc0191da47598db115748e96752145c3cea89a986
\ No newline at end of file

db/structure.sql

@@ -19946,6 +19946,27 @@ CREATE SEQUENCE upcoming_reconciliations_id_seq
 
 ALTER SEQUENCE upcoming_reconciliations_id_seq OWNED BY upcoming_reconciliations.id;
 
+CREATE TABLE upload_states (
+    verification_started_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    verified_at timestamp with time zone,
+    upload_id bigint NOT NULL,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    verification_retry_count smallint,
+    verification_checksum bytea,
+    verification_failure text,
+    CONSTRAINT check_7396dc8591 CHECK ((char_length(verification_failure) <= 255))
+);
+
+CREATE SEQUENCE upload_states_upload_id_seq
+    START WITH 1
+    INCREMENT BY 1
+    NO MINVALUE
+    NO MAXVALUE
+    CACHE 1;
+
+ALTER SEQUENCE upload_states_upload_id_seq OWNED BY upload_states.upload_id;
+
 CREATE TABLE uploads (
     id integer NOT NULL,
     size bigint NOT NULL,
@@ -22049,6 +22070,8 @@ ALTER TABLE ONLY u2f_registrations ALTER COLUMN id SET DEFAULT nextval('u2f_regi
 
 ALTER TABLE ONLY upcoming_reconciliations ALTER COLUMN id SET DEFAULT nextval('upcoming_reconciliations_id_seq'::regclass);
 
+ALTER TABLE ONLY upload_states ALTER COLUMN upload_id SET DEFAULT nextval('upload_states_upload_id_seq'::regclass);
+
 ALTER TABLE ONLY uploads ALTER COLUMN id SET DEFAULT nextval('uploads_id_seq'::regclass);
 
 ALTER TABLE ONLY user_agent_details ALTER COLUMN id SET DEFAULT nextval('user_agent_details_id_seq'::regclass);
@@ -24022,6 +24045,9 @@ ALTER TABLE ONLY u2f_registrations
 ALTER TABLE ONLY upcoming_reconciliations
     ADD CONSTRAINT upcoming_reconciliations_pkey PRIMARY KEY (id);
 
+ALTER TABLE ONLY upload_states
+    ADD CONSTRAINT upload_states_pkey PRIMARY KEY (upload_id);
+
 ALTER TABLE ONLY uploads
     ADD CONSTRAINT uploads_pkey PRIMARY KEY (id);
 
@@ -27582,6 +27608,16 @@ CREATE UNIQUE INDEX index_unit_test_failures_unique_columns ON ci_unit_test_fail
 
 CREATE UNIQUE INDEX index_upcoming_reconciliations_on_namespace_id ON upcoming_reconciliations USING btree (namespace_id);
 
+CREATE INDEX index_upload_states_failed_verification ON upload_states USING btree (verification_retry_at NULLS FIRST) WHERE (verification_state = 3);
+
+CREATE INDEX index_upload_states_needs_verification ON upload_states USING btree (verification_state) WHERE ((verification_state = 0) OR (verification_state = 3));
+
+CREATE INDEX index_upload_states_on_upload_id ON upload_states USING btree (upload_id);
+
+CREATE INDEX index_upload_states_on_verification_state ON upload_states USING btree (verification_state);
+
+CREATE INDEX index_upload_states_pending_verification ON upload_states USING btree (verified_at NULLS FIRST) WHERE (verification_state = 0);
+
 CREATE INDEX index_uploads_on_checksum ON uploads USING btree (checksum);
 
 CREATE INDEX index_uploads_on_model_id_and_model_type ON uploads USING btree (model_id, model_type);
@@ -31095,6 +31131,9 @@ ALTER TABLE ONLY resource_iteration_events
 ALTER TABLE ONLY vulnerability_finding_evidence_requests
     ADD CONSTRAINT fk_rails_cf0f278cb0 FOREIGN KEY (vulnerability_finding_evidence_supporting_message_id) REFERENCES vulnerability_finding_evidence_supporting_messages(id) ON DELETE CASCADE;
 
+ALTER TABLE ONLY upload_states
+    ADD CONSTRAINT fk_rails_d00f153613 FOREIGN KEY (upload_id) REFERENCES uploads(id) ON DELETE CASCADE;
+
 ALTER TABLE ONLY epic_metrics
     ADD CONSTRAINT fk_rails_d071904753 FOREIGN KEY (epic_id) REFERENCES epics(id) ON DELETE CASCADE;
 

doc/administration/monitoring/prometheus/gitlab_metrics.md

@@ -272,6 +272,12 @@ configuration option in `gitlab.yml`. These metrics are served from the
 | `geo_uploads_synced`               | Gauge   | 14.1  | Number of uploads synced on secondary | `url` |
 | `geo_uploads_failed`               | Gauge   | 14.1  | Number of syncable uploads failed to sync on secondary | `url` |
 | `geo_uploads_registry`             | Gauge   | 14.1  | Number of uploads in the registry | `url` |
+| `geo_uploads_checksum_total`       | Gauge   | 14.6 | Number of uploads tried to checksum on primary | `url` |
+| `geo_uploads_checksummed`          | Gauge   | 14.6  | Number of uploads successfully checksummed on primary | `url` |
+| `geo_uploads_checksum_failed`      | Gauge   | 14.6  | Number of uploads failed to calculate the checksum on primary | `url` |
+| `geo_uploads_verification_total`   | Gauge   | 14.6 | Number of uploads verifications tried on secondary | `url` |
+| `geo_uploads_verified`             | Gauge   | 14.6 | Number of uploads verified on secondary | `url` |
+| `geo_uploads_verification_failed`  | Gauge   | 14.6 | Number of uploads verifications failed on secondary | `url` |
 | `gitlab_sli:rails_request_apdex:total` | Counter | 14.4 | The number of request-apdex measurements, [more information the development documentation](../../../development/application_slis/rails_request_apdex.md) | `endpoint_id`, `feature_category`, `request_urgency` |
 | `gitlab_sli:rails_request_apdex:success_total` | Counter | 14.4 | The number of succesful requests that met the target duration for their urgency. Devide by `gitlab_sli:rails_requests_apdex:total` to get a success ratio | `endpoint_id`, `feature_category`, `request_urgency` |
 

doc/api/geo_nodes.md

@@ -453,6 +453,13 @@ Example response:
     "uploads_failed_count": 0,
     "uploads_registry_count": null,
     "uploads_synced_in_percentage": "0.00%",
+    "uploads_checksum_total_count": 5,
+    "uploads_checksummed_count": 5,
+    "uploads_checksum_failed_count": null,
+    "uploads_verification_total_count":  null,
+    "uploads_verified_count": null,
+    "uploads_verification_failed_count": null,
+    "uploads_verified_in_percentage": "0.00%",
   },
   {
     "geo_node_id": 2,
@@ -595,6 +602,13 @@ Example response:
     "uploads_failed_count": 0,
     "uploads_registry_count": null,
     "uploads_synced_in_percentage": "0.00%",
+    "uploads_checksum_total_count": 5,
+    "uploads_checksummed_count": 5,
+    "uploads_checksum_failed_count": null,
+    "uploads_verification_total_count":  null,
+    "uploads_verified_count": null,
+    "uploads_verification_failed_count": null,
+    "uploads_verified_in_percentage": "0.00%",
   }
 ]
 ```
@@ -734,6 +748,13 @@ Example response:
   "uploads_failed_count": 0,
   "uploads_registry_count": null,
   "uploads_synced_in_percentage": "0.00%",
+  "uploads_checksum_total_count": 5,
+  "uploads_checksummed_count": 5,
+  "uploads_checksum_failed_count": null,
+  "uploads_verification_total_count":  null,
+  "uploads_verified_count": null,
+  "uploads_verification_failed_count": null,
+  "uploads_verified_in_percentage": "0.00%",
 }
 ```
 

ee/app/models/ee/upload.rb

@@ -11,14 +11,47 @@ module EE
     prepended do
       include ::Gitlab::SQL::Pattern
       include ::Gitlab::Geo::ReplicableModel
+      include ::Gitlab::Geo::VerificationState
 
       with_replicator ::Geo::UploadReplicator
 
       scope :for_model, ->(model) { where(model_id: model.id, model_type: model.class.name) }
       scope :syncable, -> { with_files_stored_locally }
+      scope :with_verification_state, ->(state) { joins(:upload_state).where(upload_states: { verification_state: verification_state_value(state) }) }
+      scope :checksummed, -> { joins(:upload_state).where.not(upload_states: { verification_checksum: nil } ) }
+      scope :not_checksummed, -> { joins(:upload_state).where(upload_states: { verification_checksum: nil } ) }
+
+      scope :available_verifiables, -> { joins(:upload_state) }
+
+      has_one :upload_state,
+              autosave: false,
+              inverse_of: :upload,
+              class_name: '::Geo::UploadState'
+
+      delegate :verification_retry_at, :verification_retry_at=,
+               :verified_at, :verified_at=,
+               :verification_checksum, :verification_checksum=,
+               :verification_failure, :verification_failure=,
+               :verification_retry_count, :verification_retry_count=,
+               :verification_state=, :verification_state,
+               :verification_started_at=, :verification_started_at,
+               to: :upload_state
+
+      after_save :save_verification_details
+
+      def verification_state_object
+        upload_state
+      end
     end
 
     class_methods do
+      extend ::Gitlab::Utils::Override
+
+      override :verification_state_table_class
+      def verification_state_table_class
+        ::Geo::UploadState
+      end
+
       # @param primary_key_in [Range, Upload] arg to pass to primary_key_in scope
       # @return [ActiveRecord::Relation<Upload>] everything that should be synced to this node, restricted by primary key
       def replicables_for_current_secondary(primary_key_in)
@@ -77,5 +110,9 @@ module EE
       # Keep empty for now. Should be addressed in future
       # by https://gitlab.com/gitlab-org/gitlab/issues/33817
     end
+
+    def upload_state
+      super || build_upload_state
+    end
   end
 end

ee/app/models/geo/upload_registry.rb

@@ -2,6 +2,7 @@
 
 class Geo::UploadRegistry < Geo::BaseRegistry
   include ::Geo::ReplicableRegistry
+  include ::Geo::VerifiableRegistry
 
   extend ::Gitlab::Utils::Override
 
@@ -77,4 +78,17 @@ class Geo::UploadRegistry < Geo::BaseRegistry
   def project
     return upload.model if upload&.model.is_a?(Project)
   end
+
+  # Returns a synchronization state based on existing attribute values
+  #
+  # It takes into account things like if a successful replication has been done
+  # if there are pending actions or existing errors
+  #
+  # @return [Symbol] :synced, :never, or :failed
+  def synchronization_state
+    return :synced if success?
+    return :never if retry_count.nil?
+
+    :failed
+  end
 end

ee/app/models/geo/upload_state.rb

+# frozen_string_literal: true
+
+module Geo
+  class UploadState < ApplicationRecord
+    self.primary_key = :upload_id
+
+    belongs_to :upload, inverse_of: :upload_state
+
+    validates :verification_failure, length: { maximum: 255 }
+    validates :verification_state, :upload, presence: true
+  end
+end

ee/app/replicators/geo/upload_replicator.rb

@@ -34,6 +34,11 @@ module Geo
       ::Geo::BatchEventCreateWorker.perform_async(events)
     end
 
+    override :verification_feature_flag_enabled?
+    def self.verification_feature_flag_enabled?
+      Feature.enabled?(:geo_upload_verification, default_enabled: :yaml)
+    end
+
     def carrierwave_uploader
       model_record.retrieve_uploader
     end

ee/config/feature_flags/development/geo_upload_verification.yml

+---
+name: geo_upload_verification
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/65921
+rollout_issue_url:
+milestone: '14.6'
+type: development
+group: group::geo
+default_enabled: false

ee/db/geo/migrate/20211101113611_prepare_file_registry_for_verification.rb

+# frozen_string_literal: true
+
+class PrepareFileRegistryForVerification < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_column :file_registry, :verified_at, :datetime_with_timezone
+    add_column :file_registry, :verification_started_at, :datetime_with_timezone
+    add_column :file_registry, :verification_retry_at, :datetime_with_timezone
+    add_column :file_registry, :verification_state, :integer, default: 0, null: false, limit: 2
+    add_column :file_registry, :verification_retry_count, :integer, default: 0, limit: 2, null: false
+    add_column :file_registry, :verification_checksum, :binary
+    add_column :file_registry, :verification_checksum_mismatched, :binary
+    add_column :file_registry, :checksum_mismatch, :boolean, default: false, null: false
+
+    # rubocop:disable Migration/AddLimitToTextColumns
+    # limit is added in 20211126312431_add_text_limit_to_file_registry_verification_failure.rb
+    add_column :file_registry, :verification_failure, :text
+    # rubocop:enable Migration/AddLimitToTextColumns
+  end
+
+  def down
+    remove_column :file_registry, :verified_at
+    remove_column :file_registry, :verification_started_at
+    remove_column :file_registry, :verification_retry_at
+    remove_column :file_registry, :verification_state
+    remove_column :file_registry, :verification_retry_count
+    remove_column :file_registry, :verification_checksum
+    remove_column :file_registry, :verification_checksum_mismatched
+    remove_column :file_registry, :checksum_mismatch
+    remove_column :file_registry, :verification_failure
+  end
+end

ee/db/geo/migrate/20211102121712_create_file_registry_verification_indexies.rb

+# frozen_string_literal: true
+
+class CreateFileRegistryVerificationIndexies < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :file_registry, :verification_retry_at, name: :file_registry_failed_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 3))"
+    # To optimize performance of UploadRegistry.needs_verification_count
+    add_concurrent_index :file_registry, :verification_state, name: :file_registry_needs_verification, where: "((state = 2)  AND (verification_state = ANY (ARRAY[0, 3])))"
+    # To optimize performance of UploadRegistry.verification_pending_batch
+    add_concurrent_index :file_registry, :verified_at, name: :file_registry_pending_verification, order: "NULLS FIRST", where: "((state = 2) AND (verification_state = 0))"
+  end
+
+  def down
+    remove_concurrent_index :file_registry, :verification_retry_at, name: :file_registry_failed_verification
+    remove_concurrent_index :file_registry, :verification_state, name: :file_registry_needs_verification
+    remove_concurrent_index :file_registry, :verified_at, name: :file_registry_pending_verification
+  end
+end

ee/db/geo/migrate/20211126312431_add_text_limit_to_file_registry_verification_failure.rb

+# frozen_string_literal: true
+
+class AddTextLimitToFileRegistryVerificationFailure < Gitlab::Database::Migration[1.0]
+  disable_ddl_transaction!
+
+  def up
+    add_text_limit :file_registry, :verification_failure, 256
+  end
+
+  def down
+    remove_text_limit :file_registry, :verification_failure
+  end
+end

ee/db/geo/schema_migrations/20211101113611

+e7bcc5f9db03ce5856ee1851de05c3369fd70fce4a14b8eba3c31c66c625f9fd
\ No newline at end of file

ee/db/geo/schema_migrations/20211102121712

+b7ce85f62fc9fdc1363ba51becd096313ed0be6c0b1472468ec3de05bb3d93be
\ No newline at end of file

ee/db/geo/schema_migrations/20211126312431

+687dc025dd98af01fa9e92fd610cacebf442d8a3039ed4abff27bfcf6453a29a
\ No newline at end of file

ee/db/geo/structure.sql

@@ -72,7 +72,17 @@ CREATE TABLE file_registry (
     missing_on_primary boolean DEFAULT false NOT NULL,
     state smallint DEFAULT 0 NOT NULL,
     last_synced_at timestamp with time zone,
-    last_sync_failure character varying(255)
+    last_sync_failure character varying(255),
+    verified_at timestamp with time zone,
+    verification_started_at timestamp with time zone,
+    verification_retry_at timestamp with time zone,
+    verification_state smallint DEFAULT 0 NOT NULL,
+    verification_retry_count smallint DEFAULT 0 NOT NULL,
+    verification_checksum bytea,
+    verification_checksum_mismatched bytea,
+    checksum_mismatch boolean DEFAULT false NOT NULL,
+    verification_failure text,
+    CONSTRAINT check_1886652634 CHECK ((char_length(verification_failure) <= 256))
 );
 
 CREATE SEQUENCE file_registry_id_seq
@@ -472,6 +482,12 @@ ALTER TABLE ONLY snippet_repository_registry
 ALTER TABLE ONLY terraform_state_version_registry
     ADD CONSTRAINT terraform_state_version_registry_pkey PRIMARY KEY (id);
 
+CREATE INDEX file_registry_failed_verification ON file_registry USING btree (verification_retry_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 3));
+
+CREATE INDEX file_registry_needs_verification ON file_registry USING btree (verification_state) WHERE ((state = 2) AND (verification_state = ANY (ARRAY[0, 3])));
+
+CREATE INDEX file_registry_pending_verification ON file_registry USING btree (verified_at NULLS FIRST) WHERE ((state = 2) AND (verification_state = 0));
+
 CREATE INDEX idx_project_registry_failed_repositories_partial ON project_registry USING btree (repository_retry_count) WHERE ((repository_retry_count > 0) OR (last_repository_verification_failure IS NOT NULL) OR repository_checksum_mismatch);
 
 CREATE INDEX idx_project_registry_on_repo_checksums_and_failure_partial ON project_registry USING btree (project_id) WHERE ((repository_verification_checksum_sha IS NULL) AND (last_repository_verification_failure IS NULL));

ee/spec/factories/geo/upload_registry.rb

@@ -23,5 +23,11 @@ FactoryBot.define do
       last_synced_at { 1.day.ago }
       retry_count { 0 }
     end
+
+    trait :verification_succeeded do
+      verification_checksum { 'e079a831cab27bcda7d81cd9b48296d0c3dd92ef' }
+      verification_state { Geo::UploadRegistry.verification_state_value(:verification_succeeded) }
+      verified_at { 5.days.ago }
+    end
   end
 end

ee/spec/factories/geo/upload_states.rb

+# frozen_string_literal: true
+
+FactoryBot.define do
+  factory :geo_upload_state, class: '::Geo::UploadState' do
+    upload
+
+    trait(:checksummed) do
+      verification_checksum { 'abc' }
+    end
+
+    trait(:checksum_failure) do
+      verification_failure { 'Could not calculate the checksum' }
+    end
+  end
+end

ee/spec/factories/uploads.rb

@@ -7,5 +7,22 @@ FactoryBot.modify do
       mount_point { :file }
       uploader { ::IssuableMetricImageUploader.name }
     end
+
+    trait(:verification_succeeded) do
+      with_file
+      verification_checksum { 'abc' }
+      verification_state { Upload.verification_state_value(:verification_succeeded) }
+    end
+
+    trait(:verification_failed) do
+      with_file
+      verification_failure { 'Could not calculate the checksum' }
+      verification_state { Upload.verification_state_value(:verification_failed) }
+    end
+
+    trait(:verification_pending) do
+      with_file
+      verification_state { Upload.verification_state_value(:verification_pending) }
+    end
   end
 end

ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -153,16 +153,16 @@
     "replication_slots_used_in_percentage",
     "replication_slots_max_retained_wal_bytes",
     "uploads_count",
-    "uploads_synced_count",
-    "uploads_failed_count",
-    "uploads_registry_count",
-    "uploads_synced_in_percentage",
     "uploads_checksum_total_count",
     "uploads_checksummed_count",
     "uploads_checksum_failed_count",
-    "uploads_verification_failed_count",
+    "uploads_synced_count",
+    "uploads_failed_count",
+    "uploads_registry_count",
     "uploads_verification_total_count",
     "uploads_verified_count",
+    "uploads_verification_failed_count",
+    "uploads_synced_in_percentage",
     "uploads_verified_in_percentage",
     "git_fetch_event_count_weekly",
     "git_push_event_count_weekly",
@@ -323,16 +323,16 @@
     "repositories_verified_in_percentage": { "type": "string" },
     "repositories_checksum_mismatch_count": { "type": ["integer", "null"] },
     "uploads_count": { "type": ["integer", "null"] },
+    "uploads_checksum_total_count": { "type": ["integer", "null"] },
+    "uploads_checksummed_count": { "type": ["integer", "null"] },
+    "uploads_checksum_failed_count": { "type": ["integer", "null"] },
     "uploads_synced_count": { "type": ["integer", "null"] },
     "uploads_failed_count": { "type": ["integer", "null"] },
     "uploads_registry_count": { "type": ["integer", "null"] },
-    "uploads_synced_in_percentage": { "type": "string" },
-    "uploads_checksummed_count": { "type": ["integer", "null"] },
-    "uploads_checksum_failed_count": { "type": ["integer", "null"] },
-    "uploads_checksum_total_count": { "type": ["integer", "null"] },
-    "uploads_verification_failed_count": { "type": ["integer", "null"] },
     "uploads_verification_total_count": { "type": ["integer", "null"] },
     "uploads_verified_count": { "type": ["integer", "null"] },
+    "uploads_verification_failed_count": { "type": ["integer", "null"] },
+    "uploads_synced_in_percentage": { "type": "string" },
     "uploads_verified_in_percentage": { "type": "string" },
     "wikis_verified_count": { "type": ["integer", "null"] },
     "wikis_verification_failed_count": { "type": ["integer", "null"] },

ee/spec/lib/ee/gitlab/usage_data_non_sql_metrics_spec.rb

@@ -15,7 +15,7 @@ RSpec.describe Gitlab::UsageDataNonSqlMetrics do
         described_class.uncached_data
       end
 
-      expect(recorder.count).to eq(59)
+      expect(recorder.count).to eq(63)
     end
   end
 end

ee/spec/models/geo/upload_registry_spec.rb

@@ -11,3 +11,14 @@ RSpec.describe Geo::UploadRegistry, :geo, type: :model do
 
   include_examples 'a Geo framework registry'
 end
+
+RSpec.describe Geo::UploadRegistry, :geo, type: :model do
+  let_it_be(:registry) { create(:geo_upload_registry) }
+
+  specify 'factory is valid' do
+    expect(registry).to be_valid
+  end
+
+  include_examples 'a Geo framework registry'
+  include_examples 'a Geo verifiable registry'
+end

ee/spec/models/geo/upload_state_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Geo::UploadState, :geo, type: :model do
+  it { is_expected.to belong_to(:upload).inverse_of(:upload_state) }
+end

ee/spec/models/geo_node_status_spec.rb

@@ -1302,35 +1302,25 @@ RSpec.describe GeoNodeStatus, :geo do
     end
 
     context 'on the secondary' do
-      it 'calls JobArtifactRegistryFinder#registry_count' do
-        expect_any_instance_of(Geo::JobArtifactRegistryFinder).to receive(:registry_count).twice
+      it 'returns data from the deprecated field if it is not defined in the status field' do
+        subject.write_attribute(:projects_count, 10)
+        subject.status = {}
 
-        subject
+        expect(subject.projects_count).to eq 10
       end
-    end
-
-    context 'backward compatibility when counters stored in separate columns' do
-      describe '#projects_count' do
-        it 'returns data from the deprecated field if it is not defined in the status field' do
-          subject.write_attribute(:projects_count, 10)
-          subject.status = {}
-
-          expect(subject.projects_count).to eq 10
-        end
 
-        it 'sets data in the new status field' do
-          subject.projects_count = 10
+      it 'sets data in the new status field' do
+        subject.projects_count = 10
 
-          expect(subject.projects_count).to eq 10
-        end
+        expect(subject.projects_count).to eq 10
+      end
 
-        it 'uses column counters when calculates percents using attr_in_percentage' do
-          subject.write_attribute(:design_repositories_count, 10)
-          subject.write_attribute(:design_repositories_synced_count, 5)
-          subject.status = {}
+      it 'uses column counters when calculates percents using attr_in_percentage' do
+        subject.write_attribute(:design_repositories_count, 10)
+        subject.write_attribute(:design_repositories_synced_count, 5)
+        subject.status = {}
 
-          expect(subject.design_repositories_synced_in_percentage).to be_within(0.0001).of(50)
-        end
+        expect(subject.design_repositories_synced_in_percentage).to be_within(0.0001).of(50)
       end
     end
 

ee/spec/models/upload_spec.rb

@@ -6,6 +6,13 @@ RSpec.describe Upload do
   include EE::GeoHelpers
   using RSpec::Parameterized::TableSyntax
 
+  it { is_expected.to have_one(:upload_state).inverse_of(:upload).class_name('Geo::UploadState') }
+
+  include_examples 'a replicable model with a separate table for verification state' do
+    let(:verifiable_model_record) { build(:upload) }
+    let(:unverifiable_model_record) { build(:upload, store: ObjectStorage::Store::REMOTE) }
+  end
+
   describe '.replicables_for_current_secondary' do
     # Selective sync is configured relative to the upload's model. Take care not
     # to specify a model_factory that contradicts factory.

ee/spec/replicators/geo/upload_replicator_spec.rb

@@ -6,6 +6,7 @@ RSpec.describe Geo::UploadReplicator do
   let(:model_record) { create(:upload, :with_file) }
 
   include_examples 'a blob replicator'
+  include_examples 'a verifiable replicator'
 
   describe '.bulk_create_delete_events_async' do
     let(:deleted_upload) do

ee/spec/services/geo/file_upload_service_spec.rb

@@ -9,6 +9,7 @@ RSpec.describe Geo::FileUploadService do
 
   before do
     stub_current_geo_node(node)
+    stub_feature_flags(geo_upload_verification: false)
   end
 
   describe '#retriever' do

ee/spec/services/projects/cleanup_service_spec.rb

@@ -14,6 +14,7 @@ RSpec.describe Projects::CleanupService do
   describe '#execute' do
     before do
       stub_current_geo_node(primary)
+      stub_feature_flags(geo_upload_verification: false)
     end
 
     it 'sends a Geo notification about the update on success' do

ee/spec/support/shared_examples/models/concerns/replicable_model_with_separate_table_shared_examples.rb

@@ -12,6 +12,47 @@
 RSpec.shared_examples 'a replicable model with a separate table for verification state' do
   include EE::GeoHelpers
 
+  describe '.with_verification_state' do
+    let(:verification_model_class) { verifiable_model_record.class }
+
+    it 'returns records with given scope' do
+      expect(verification_model_class.with_verification_state(:verification_succeeded).size).to eq(0)
+
+      verifiable_model_record.verification_failed_with_message!('Test')
+
+      expect(verification_model_class.with_verification_state(:verification_failed).first).to eq verifiable_model_record
+    end
+  end
+
+  describe '.checksummed' do
+    let(:verification_model_class) { verifiable_model_record.class }
+
+    it 'returns records with given scope' do
+      expect(verification_model_class.checksummed.size).to eq(0)
+
+      verifiable_model_record.verification_started!
+      verifiable_model_record.verification_succeeded_with_checksum!('checksum', Time.now)
+
+      expect(verification_model_class.checksummed.first).to eq verifiable_model_record
+    end
+  end
+
+  describe '.not_checksummed' do
+    let(:verification_model_class) { verifiable_model_record.class }
+
+    it 'returns records with given scope' do
+      verifiable_model_record.verification_started!
+      verifiable_model_record.verification_failed_with_message!('checksum error')
+
+      expect(verification_model_class.not_checksummed.first).to eq verifiable_model_record
+
+      verifiable_model_record.verification_started!
+      verifiable_model_record.verification_succeeded_with_checksum!('checksum', Time.now)
+
+      expect(verification_model_class.not_checksummed.size).to eq(0)
+    end
+  end
+
   describe '#save_verification_details' do
     let(:verification_state_table_class) { verifiable_model_record.class.verification_state_table_class }
 

ee/spec/support/shared_examples/models/concerns/verifiable_replicator_shared_examples.rb

@@ -474,6 +474,8 @@ RSpec.shared_examples 'a verifiable replicator' do
       end
 
       it 'creates checksum_succeeded event' do
+        model_record
+
         expect { replicator.handle_after_checksum_succeeded }.to change { ::Geo::Event.count }.by(1)
         expect(::Geo::Event.last.event_name).to eq 'checksum_succeeded'
       end

lib/gitlab/database/gitlab_schemas.yml

@@ -490,6 +490,7 @@ trending_projects: :gitlab_main
 u2f_registrations: :gitlab_main
 upcoming_reconciliations: :gitlab_main
 uploads: :gitlab_main
+upload_states: :gitlab_main
 user_agent_details: :gitlab_main
 user_callouts: :gitlab_main
 user_canonical_emails: :gitlab_main

spec/requests/api/project_import_spec.rb

@@ -47,7 +47,7 @@ RSpec.describe API::ProjectImport do
     it 'executes a limited number of queries' do
       control_count = ActiveRecord::QueryRecorder.new { subject }.count
 
-      expect(control_count).to be <= 102
+      expect(control_count).to be <= 104
     end
 
     it 'schedules an import using a namespace' do