Geo - Repository verification on secondary node

app/workers/all_queues.yml

@@ -113,6 +113,7 @@
 - cronjob:geo_prune_event_log
 - cronjob:geo_repository_sync
 - cronjob:geo_repository_verification_primary_batch
+- cronjob:geo_repository_verification_secondary_scheduler
 - cronjob:geo_sidekiq_cron_config
 - cronjob:historical_data
 - cronjob:ldap_all_groups_sync
@@ -133,6 +134,7 @@
 - geo:geo_repository_shard_sync
 - geo:geo_repository_verification_primary_shard
 - geo:geo_repository_verification_primary_single
+- geo:geo_repository_verification_secondary_single
 
 - object_storage_upload
 - object_storage:object_storage_background_move

config/initializers/1_settings.rb

@@ -479,6 +479,9 @@ Settings.cron_jobs['geo_prune_event_log_worker']['job_class'] ||= 'Geo::PruneEve
 Settings.cron_jobs['geo_repository_verification_primary_batch_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['geo_repository_verification_primary_batch_worker']['cron'] ||= '*/1 * * * *'
 Settings.cron_jobs['geo_repository_verification_primary_batch_worker']['job_class'] ||= 'Geo::RepositoryVerification::Primary::BatchWorker'
+Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker']['cron'] ||= '*/1 * * * *'
+Settings.cron_jobs['geo_repository_verification_secondary_scheduler_worker']['job_class'] ||= 'Geo::RepositoryVerification::Secondary::SchedulerWorker'
 Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'

db/schema.rb

@@ -1047,6 +1047,10 @@ ActiveRecord::Schema.define(version: 20180307164427) do
     t.integer "job_artifacts_failed_count"
     t.string "version"
     t.string "revision"
+    t.integer "repositories_verified_count"
+    t.integer "repositories_verification_failed_count"
+    t.integer "wikis_verified_count"
+    t.integer "wikis_verification_failed_count"
   end
 
   add_index "geo_node_statuses", ["geo_node_id"], name: "index_geo_node_statuses_on_geo_node_id", unique: true, using: :btree

ee/app/assets/javascripts/geo_nodes/components/geo_node_details.vue

@@ -203,6 +203,22 @@
       </li>
       <template v-if="showAdvanceItems">
         <template v-if="node.primary">
+          <geo-node-detail-item
+            :item-title="s__('GeoNodes|Repositories checksummed:')"
+            :success-label="s__('GeoNodes|Checksummed')"
+            :neutral-label="s__('GeoNodes|Not checksummed')"
+            :failure-label="s__('GeoNodes|Failed')"
+            :item-value="nodeDetails.verifiedRepositories"
+            :item-value-type="valueType.GRAPH"
+          />
+          <geo-node-detail-item
+            :item-title="s__('GeoNodes|Wikis checksummed:')"
+            :success-label="s__('GeoNodes|Checksummed')"
+            :neutral-label="s__('GeoNodes|Not checksummed')"
+            :failure-label="s__('GeoNodes|Failed')"
+            :item-value="nodeDetails.verifiedWikis"
+            :item-value-type="valueType.GRAPH"
+          />
           <geo-node-detail-item
             :item-title="s__('GeoNodes|Replication slots:')"
             :success-label="s__('GeoNodes|Used slots')"
@@ -219,6 +235,22 @@
           />
         </template>
         <template v-else>
+          <geo-node-detail-item
+            :item-title="s__('GeoNodes|Repository checksums verified:')"
+            :success-label="s__('GeoNodes|Verified')"
+            :neutral-label="s__('GeoNodes|Unverified')"
+            :failure-label="s__('GeoNodes|Failed')"
+            :item-value="nodeDetails.verifiedRepositories"
+            :item-value-type="valueType.GRAPH"
+          />
+          <geo-node-detail-item
+            :item-title="s__('GeoNodes|Wiki checksums verified:')"
+            :success-label="s__('GeoNodes|Verified')"
+            :neutral-label="s__('GeoNodes|Unverified')"
+            :failure-label="s__('GeoNodes|Failed')"
+            :item-value="nodeDetails.verifiedWikis"
+            :item-value-type="valueType.GRAPH"
+          />
           <geo-node-detail-item
             css-class="node-detail-value-bold"
             :item-title="s__('GeoNodes|Database replication lag:')"

ee/app/assets/javascripts/geo_nodes/store/geo_nodes_store.js

@@ -87,6 +87,16 @@ export default class GeoNodesStore {
         successCount: rawNodeDetails.wikis_synced_count || 0,
         failureCount: rawNodeDetails.wikis_failed_count || 0,
       },
+      verifiedRepositories: {
+        totalCount: rawNodeDetails.repositories_count || 0,
+        successCount: rawNodeDetails.repositories_verified_count || 0,
+        failureCount: rawNodeDetails.repositories_verification_failed_count || 0,
+      },
+      verifiedWikis: {
+        totalCount: rawNodeDetails.wikis_count || 0,
+        successCount: rawNodeDetails.wikis_verified_count || 0,
+        failureCount: rawNodeDetails.wikis_verification_failed_count || 0,
+      },
       lfs: {
         totalCount: rawNodeDetails.lfs_objects_count || 0,
         successCount: rawNodeDetails.lfs_objects_synced_count || 0,

ee/app/controllers/ee/admin/logs_controller.rb

@@ -6,7 +6,8 @@ module EE::Admin::LogsController
   def loggers
     strong_memoize(:loggers) do
       super + [
-        Gitlab::GeoLogger
+        Gitlab::GeoLogger,
+        Gitlab::Geo::RepositoryVerificationLogger
       ]
     end
   end

ee/app/finders/geo/project_registry_finder.rb

@@ -39,14 +39,58 @@ module Geo
     end
 
     def find_failed_project_registries(type = nil)
+      if selective_sync?
+        legacy_find_filtered_failed_projects(type)
+      else
+        find_filtered_failed_project_registries(type)
+      end
+    end
+
+    def count_verified_repositories
       relation =
-        if selective_sync?
-          legacy_find_filtered_failed_projects(type)
+        if use_legacy_queries?
+          legacy_find_verified_repositories
         else
-          find_filtered_failed_project_registries(type)
+          find_verified_repositories
         end
 
-      relation
+      relation.count
+    end
+
+    def count_verified_wikis
+      relation =
+        if use_legacy_queries?
+          legacy_find_verified_wikis
+        else
+          find_verified_wikis
+        end
+
+      relation.count
+    end
+
+    def count_verification_failed_repositories
+      find_verification_failed_project_registries('repository').count
+    end
+
+    def count_verification_failed_wikis
+      find_verification_failed_project_registries('wiki').count
+    end
+
+    def find_verification_failed_project_registries(type = nil)
+      if use_legacy_queries?
+        legacy_find_filtered_verification_failed_projects(type)
+      else
+        find_filtered_verification_failed_project_registries(type)
+      end
+    end
+
+    # find all registries that need a repository or wiki verified
+    def find_registries_to_verify
+      if use_legacy_queries?
+        legacy_find_registries_to_verify
+      else
+        fdw_find_registries_to_verify
+      end
     end
 
     def find_unsynced_projects(batch_size:)
@@ -77,6 +121,14 @@ module Geo
       Geo::ProjectRegistry.synced_repos
     end
 
+    def find_verified_repositories
+      Geo::ProjectRegistry.verified_repos
+    end
+
+    def find_verified_wikis
+      Geo::ProjectRegistry.verified_wikis
+    end
+
     def find_filtered_failed_project_registries(type = nil)
       case type
       when 'repository'
@@ -88,17 +140,52 @@ module Geo
       end
     end
 
+    def find_filtered_verification_failed_project_registries(type = nil)
+      case type
+      when 'repository'
+        Geo::ProjectRegistry.verification_failed_repos
+      when 'wiki'
+        Geo::ProjectRegistry.verification_failed_wikis
+      else
+        Geo::ProjectRegistry.verification_failed
+      end
+    end
+
+    def conditions_for_verification(type, use_fdw = true)
+      last_verification_failed  = "last_#{type}_verification_failed".to_sym
+      verification_checksum     = "#{type}_verification_checksum".to_sym
+      last_verification_at      = "last_#{type}_verification_at".to_sym
+
+      state_arel = use_fdw ? fdw_repository_state_arel : legacy_repository_state_arel
+
+      # primary verification did not fail
+      primary_verification_not_failed = state_arel[last_verification_failed].eq(false)
+
+      # primary checksum is not NULL
+      primary_has_checksum = state_arel[verification_checksum].not_eq(nil)
+
+      # primary was verified later than the secondary verification
+      primary_recently_verified = state_arel[last_verification_at].gt(registry_arel[last_verification_at])
+                                    .or(registry_arel[last_verification_at].eq(nil))
+
+      # secondary verification failed and the last verification was over 24.hours.ago
+      # this allows us to retry any verification failures if they haven't already corrected themselves
+      secondary_failure_period = registry_arel[last_verification_at].lt(24.hours.ago)
+                                   .and(registry_arel[last_verification_failed].eq(true))
+
+      primary_verification_not_failed
+        .and(primary_has_checksum)
+        .and(primary_recently_verified)
+        .or(secondary_failure_period)
+    end
+
     #
     # FDW accessors
     #
 
-    def fdw_table
-      Geo::Fdw::Project.table_name
-    end
-
     # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
     def fdw_find_unsynced_projects
-      Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
+      Geo::Fdw::Project.joins("LEFT OUTER JOIN project_registry ON project_registry.project_id = #{fdw_project_table}.id")
         .where(project_registry: { project_id: nil })
     end
 
@@ -121,11 +208,19 @@ module Geo
 
     # @return [ActiveRecord::Relation<Geo::Fdw::Project>]
     def fdw_find_projects_updated_recently
-      Geo::Fdw::Project.joins("INNER JOIN project_registry ON project_registry.project_id = #{fdw_table}.id")
+      Geo::Fdw::Project.joins("INNER JOIN project_registry ON project_registry.project_id = #{fdw_project_table}.id")
           .merge(Geo::ProjectRegistry.dirty)
           .merge(Geo::ProjectRegistry.retry_due)
     end
 
+    # find all registries that need a repository or wiki verified
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of registries that need verification
+    def fdw_find_registries_to_verify
+      Geo::ProjectRegistry
+        .joins("LEFT OUTER JOIN #{fdw_repository_state_table} ON #{fdw_repository_state_table}.project_id = project_registry.project_id")
+        .where(conditions_for_verification(:repository, true).or(conditions_for_verification(:wiki, true)))
+    end
+
     #
     # Legacy accessors (non FDW)
     #
@@ -176,6 +271,16 @@ module Geo
       )
     end
 
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of verified projects
+    def legacy_find_verified_repositories
+      legacy_find_project_registries(Geo::ProjectRegistry.verified_repos)
+    end
+
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of verified projects
+    def legacy_find_verified_wikis
+      legacy_find_project_registries(Geo::ProjectRegistry.verified_wikis)
+    end
+
     # @return [ActiveRecord::Relation<Project>] list of synced projects
     def legacy_find_project_registries(project_registries)
       legacy_inner_join_registry_ids(
@@ -194,5 +299,65 @@ module Geo
         foreign_key: :project_id
       )
     end
+
+    # @return [ActiveRecord::Relation<Project>] list of projects that verification has failed
+    def legacy_find_filtered_verification_failed_projects(type = nil)
+      legacy_inner_join_registry_ids(
+        find_filtered_verification_failed_project_registries(type),
+        current_node.projects.pluck(:id),
+        Geo::ProjectRegistry,
+        foreign_key: :project_id
+      )
+    end
+
+    # @return [ActiveRecord::Relation<Geo::ProjectRegistry>] list of registries that need verification
+    def legacy_find_registries_to_verify
+      registries = Geo::ProjectRegistry
+        .pluck(:project_id, :last_repository_verification_at, :last_wiki_verification_at,
+               :last_repository_verification_failed, :last_wiki_verification_failed)
+
+      return Geo::ProjectRegistry.none if registries.empty?
+
+      id_and_values = registries.map do |project_id, repo_at, wiki_at, repo_failed, wiki_failed|
+        "(#{project_id}, to_timestamp(#{repo_at.to_i}), to_timestamp(#{wiki_at.to_i}),
+          #{quote_value(repo_failed)}, #{quote_value(wiki_failed)})"
+      end
+
+      joined_relation = ProjectRepositoryState.joins(<<~SQL)
+        INNER JOIN
+        (VALUES #{id_and_values.join(',')})
+        project_registry(project_id, last_repository_verification_at, last_wiki_verification_at,
+                         last_repository_verification_failed, last_wiki_verification_failed)
+        ON #{ProjectRepositoryState.table_name}.project_id = project_registry.project_id
+      SQL
+
+      project_ids = joined_relation
+        .where(conditions_for_verification(:repository, false).or(conditions_for_verification(:wiki, false)))
+        .pluck(:project_id)
+
+      ::Geo::ProjectRegistry.where(project_id: project_ids)
+    end
+
+    private
+
+    def registry_arel
+      Geo::ProjectRegistry.arel_table
+    end
+
+    def fdw_repository_state_arel
+      Geo::Fdw::ProjectRepositoryState.arel_table
+    end
+
+    def legacy_repository_state_arel
+      ::ProjectRepositoryState.arel_table
+    end
+
+    def fdw_project_table
+      Geo::Fdw::Project.table_name
+    end
+
+    def fdw_repository_state_table
+      Geo::Fdw::ProjectRepositoryState.table_name
+    end
   end
 end

ee/app/finders/geo/repository_verification_finder.rb

@@ -19,6 +19,22 @@ module Geo
        .limit(batch_size)
     end
 
+    def count_verified_repositories
+      Project.verified_repos.count
+    end
+
+    def count_verified_wikis
+      Project.verified_wikis.count
+    end
+
+    def count_verification_failed_repositories
+      Project.verification_failed_repos.count
+    end
+
+    def count_verification_failed_wikis
+      Project.verification_failed_wikis.count
+    end
+
     protected
 
     def projects_table

ee/app/models/ee/project.rb

@@ -54,7 +54,12 @@ module EE
       end
 
       scope :with_remote_mirrors, -> { joins(:remote_mirrors).where(remote_mirrors: { enabled: true }).distinct }
-      scope :with_wiki_enabled, -> { with_feature_enabled(:wiki) }
+      scope :with_wiki_enabled,   -> { with_feature_enabled(:wiki) }
+
+      scope :verified_repos, -> { joins(:repository_state).merge(ProjectRepositoryState.verified_repos) }
+      scope :verified_wikis, -> { joins(:repository_state).merge(ProjectRepositoryState.verified_wikis) }
+      scope :verification_failed_repos, -> { joins(:repository_state).merge(ProjectRepositoryState.verification_failed_repos) }
+      scope :verification_failed_wikis, -> { joins(:repository_state).merge(ProjectRepositoryState.verification_failed_wikis) }
 
       delegate :shared_runners_minutes, :shared_runners_seconds, :shared_runners_seconds_last_reset,
         to: :statistics, allow_nil: true

ee/app/models/geo/fdw/project_repository_state.rb

+module Geo
+  module Fdw
+    class ProjectRepositoryState < ::Geo::BaseFdw
+      self.table_name = Gitlab::Geo::Fdw.table('project_repository_states')
+    end
+  end
+end

ee/app/models/geo/project_registry.rb

 class Geo::ProjectRegistry < Geo::BaseRegistry
+  include ::EachBatch
+
   belongs_to :project
 
   validates :project, presence: true, uniqueness: true
@@ -6,6 +8,8 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
   scope :dirty, -> { where(arel_table[:resync_repository].eq(true).or(arel_table[:resync_wiki].eq(true))) }
   scope :failed_repos, -> { where(arel_table[:repository_retry_count].gt(0)) }
   scope :failed_wikis, -> { where(arel_table[:wiki_retry_count].gt(0)) }
+  scope :verification_failed_repos, -> { where(arel_table[:last_repository_verification_failed].eq(true)) }
+  scope :verification_failed_wikis, -> { where(arel_table[:last_wiki_verification_failed].eq(true)) }
 
   def self.failed
     repository_sync_failed = arel_table[:repository_retry_count].gt(0)
@@ -14,6 +18,13 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     where(repository_sync_failed.or(wiki_sync_failed))
   end
 
+  def self.verification_failed
+    repository_verification_failed = arel_table[:last_repository_verification_failed].eq(true)
+    wiki_verification_failed = arel_table[:last_wiki_verification_failed].eq(true)
+
+    where(repository_verification_failed.or(wiki_verification_failed))
+  end
+
   def self.retry_due
     where(
       arel_table[:repository_retry_at].lt(Time.now)
@@ -33,6 +44,16 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
         .where(resync_wiki: false)
   end
 
+  def self.verified_repos
+    where.not(last_repository_verification_at: nil, repository_verification_checksum: nil)
+        .where(last_repository_verification_failed: false)
+  end
+
+  def self.verified_wikis
+    where.not(last_wiki_verification_at: nil, wiki_verification_checksum: nil)
+        .where(last_wiki_verification_failed: false)
+  end
+
   def repository_sync_due?(scheduled_time)
     never_synced_repository? || repository_sync_needed?(scheduled_time)
   end
@@ -41,6 +62,22 @@ class Geo::ProjectRegistry < Geo::BaseRegistry
     project.wiki_enabled? && (never_synced_wiki? || wiki_sync_needed?(scheduled_time))
   end
 
+  delegate :repository_state, to: :project
+  delegate :repository_verification_checksum, :last_repository_verification_at,
+           :wiki_verification_checksum, :last_wiki_verification_at,
+           to: :repository_state, allow_nil: true, prefix: :project
+
+  def repository_path(type)
+    repo_path = project.disk_path
+
+    case type
+    when :repository
+      repo_path
+    when :wiki
+      "#{repo_path}.wiki"
+    end
+  end
+
   private
 
   def never_synced_repository?

ee/app/models/geo_node_status.rb

@@ -24,6 +24,10 @@ class GeoNodeStatus < ActiveRecord::Base
     wikis_count: 'Total number of wikis available on primary',
     wikis_synced_count: 'Number of wikis synced on secondary',
     wikis_failed_count: 'Number of wikis failed to sync on secondary',
+    repositories_verified_count: 'Number of repositories verified on secondary',
+    repositories_verification_failed_count: 'Number of repositories failed to verify on secondary',
+    wikis_verified_count: 'Number of wikis verified on secondary',
+    wikis_verification_failed_count: 'Number of wikis failed to verify on secondary',
     lfs_objects_count: 'Total number of local LFS objects available on primary',
     lfs_objects_synced_count: 'Number of local LFS objects synced on secondary',
     lfs_objects_failed_count: 'Number of local LFS objects failed to sync on secondary',
@@ -136,6 +140,10 @@ class GeoNodeStatus < ActiveRecord::Base
       self.replication_slots_count = geo_node.replication_slots_count
       self.replication_slots_used_count = geo_node.replication_slots_used_count
       self.replication_slots_max_retained_wal_bytes = geo_node.replication_slots_max_retained_wal_bytes
+      self.repositories_verified_count = repository_verification_finder.count_verified_repositories
+      self.repositories_verification_failed_count = repository_verification_finder.count_verification_failed_repositories
+      self.wikis_verified_count = repository_verification_finder.count_verified_wikis
+      self.wikis_verification_failed_count = repository_verification_finder.count_verification_failed_wikis
     end
   end
 
@@ -148,6 +156,10 @@ class GeoNodeStatus < ActiveRecord::Base
       self.repositories_failed_count = projects_finder.count_failed_repositories
       self.wikis_synced_count = projects_finder.count_synced_wikis
       self.wikis_failed_count = projects_finder.count_failed_wikis
+      self.repositories_verified_count = projects_finder.count_verified_repositories
+      self.repositories_verification_failed_count = projects_finder.count_verification_failed_repositories
+      self.wikis_verified_count = projects_finder.count_verified_wikis
+      self.wikis_verification_failed_count = projects_finder.count_verification_failed_wikis
       self.lfs_objects_synced_count = lfs_objects_finder.count_synced_lfs_objects
       self.lfs_objects_failed_count = lfs_objects_finder.count_failed_lfs_objects
       self.job_artifacts_synced_count = job_artifacts_finder.count_synced_job_artifacts
@@ -199,6 +211,14 @@ class GeoNodeStatus < ActiveRecord::Base
     calc_percentage(wikis_count, wikis_synced_count)
   end
 
+  def repositories_verified_in_percentage
+    calc_percentage(repositories_count, repositories_verified_count)
+  end
+
+  def wikis_verified_in_percentage
+    calc_percentage(wikis_count, wikis_verified_count)
+  end
+
   def lfs_objects_synced_in_percentage
     calc_percentage(lfs_objects_count, lfs_objects_synced_count)
   end
@@ -277,6 +297,10 @@ class GeoNodeStatus < ActiveRecord::Base
     @projects_finder ||= Geo::ProjectRegistryFinder.new(current_node: geo_node)
   end
 
+  def repository_verification_finder
+    @repository_verification_finder ||= Geo::RepositoryVerificationFinder.new
+  end
+
   def calc_percentage(total, count)
     return 0 if !total.present? || total.zero?
 

ee/app/models/project_repository_state.rb

@@ -8,6 +8,9 @@ class ProjectRepositoryState < ActiveRecord::Base
 
   validates :project, presence: true, uniqueness: true
 
+  scope :verification_failed_repos, -> { where(arel_table[:last_repository_verification_failed].eq(true)) }
+  scope :verification_failed_wikis, -> { where(arel_table[:last_wiki_verification_failed].eq(true)) }
+
   def repository_checksum_outdated?(timestamp)
     repository_verification_checksum.nil? || recalculate_repository_checksum?(timestamp)
   end
@@ -18,7 +21,15 @@ class ProjectRepositoryState < ActiveRecord::Base
     wiki_verification_checksum.nil? || recalculate_wiki_checksum?(timestamp)
   end
 
-  private
+  def self.verified_repos
+    where.not(repository_verification_checksum: nil)
+    .where(last_repository_verification_failed: false)
+  end
+
+  def self.verified_wikis
+    where.not(wiki_verification_checksum: nil)
+    .where(last_wiki_verification_failed: false)
+  end
 
   def recalculate_repository_checksum?(timestamp)
     last_repository_verification_at.nil? || timestamp > last_repository_verification_at

ee/app/services/geo/base_sync_service.rb

@@ -112,6 +112,11 @@ module Geo
         attrs["last_#{type}_synced_at"] = started_at
         attrs["#{type}_retry_count"] = retry_count + 1
         attrs["#{type}_retry_at"] = next_retry_time(attrs["#{type}_retry_count"])
+
+        # indicate that repository verification needs to be done again
+        attrs["#{type}_verification_checksum"] = nil
+        attrs["last_#{type}_verification_at"] = nil
+        attrs["last_#{type}_verification_failure"] = nil
       end
 
       if finished_at

ee/app/services/geo/repository_verify_secondary_service.rb

+# rubocop:disable GitlabSecurity/PublicSend
+
+module Geo
+  class RepositoryVerifySecondaryService
+    include Gitlab::Geo::RepositoryVerificationLogHelpers
+
+    delegate :project, to: :registry
+
+    def initialize(registry, type)
+      @registry = registry
+      @type     = type.to_sym
+    end
+
+    def execute
+      return unless Gitlab::Geo.geo_database_configured?
+      return unless Gitlab::Geo.secondary?
+      return unless should_verify_checksum?
+
+      verify_checksum
+    end
+
+    # This is primarily a guard method, to reduce the chance of false failures (which could happen
+    # for repositories that change very rapidly)
+    def should_verify_checksum?
+      primary_checksum                  = registry.repository_state.public_send("#{type}_verification_checksum")
+      secondary_checksum                = registry.public_send("#{type}_verification_checksum")
+      primary_last_verification_at      = registry.repository_state.public_send("last_#{type}_verification_at")
+      secondary_last_verification_at    = registry.public_send("last_#{type}_verification_at") || Time.at(0)
+      secondary_last_successful_sync_at = registry.public_send("last_#{type}_successful_sync_at")
+
+      # primary repository was verified (even if checksum is nil).
+      # note: we allow a nil primary checksum so that we will run through the checksum
+      # and set the verification date on the secondary.  Otherwise, we'll keep revisiting
+      # this record over and over.
+      return false if primary_last_verification_at.nil?
+
+      # secondary repository checksum does not equal the primary repository checksum
+      return false if secondary_checksum == primary_checksum && !primary_checksum.nil?
+
+      # primary was verified later than the secondary verification
+      return false if primary_last_verification_at < secondary_last_verification_at
+
+      # secondary repository was successfully synced after the last secondary verification
+      return false if secondary_last_successful_sync_at.nil? || secondary_last_successful_sync_at < secondary_last_verification_at
+
+      true
+    end
+
+    private
+
+    attr_reader :registry, :type
+
+    def verify_checksum
+      checksum = calculate_checksum(project.repository_storage, repository_path)
+
+      if mismatch?(checksum)
+        record_status(error_msg: "#{type.to_s.capitalize} checksum mismatch: #{repository_path}")
+      else
+        record_status(checksum: checksum)
+      end
+    rescue ::Gitlab::Git::Repository::NoRepository, ::Gitlab::Git::Checksum::Failure, Timeout::Error => e
+      record_status(error_msg: "Error verifying #{type.to_s.capitalize} checksum: #{repository_path}", exception: e)
+    end
+
+    def mismatch?(checksum)
+      checksum != registry.public_send("project_#{type}_verification_checksum")
+    end
+
+    def calculate_checksum(storage, relative_path)
+      Gitlab::Git::Checksum.new(storage, relative_path).calculate
+    end
+
+    # note: the `last_#{type}_verification_at` is always set, indicating that was the
+    # time that we _did_ a verification, success or failure
+    def record_status(checksum: nil, error_msg: nil, exception: nil, details: {})
+      attrs = {
+        "#{type}_verification_checksum"     => checksum,
+        "last_#{type}_verification_at"      => DateTime.now,
+        "last_#{type}_verification_failure" => nil,
+        "last_#{type}_verification_failed"  => false
+      }
+
+      if error_msg
+        attrs["last_#{type}_verification_failed"]  = true
+        attrs["last_#{type}_verification_failure"] = error_msg
+
+        log_error(error_msg, exception, type: type, repository_path: repository_path, full_path: path_to_repo)
+      end
+
+      registry.update!(attrs)
+    end
+
+    def repository_path
+      registry.repository_path(type)
+    end
+
+    def path_to_repo
+      case type
+      when :repository
+        project.repository.path_to_repo
+      when :wiki
+        project.wiki.repository.path_to_repo
+      end
+    end
+  end
+end

ee/app/workers/geo/repository_verification/secondary/scheduler_worker.rb

+module Geo
+  module RepositoryVerification
+    module Secondary
+      class SchedulerWorker < Geo::Scheduler::SecondaryWorker
+        include CronjobQueue
+
+        MAX_CAPACITY = 1000
+
+        def perform
+          return unless Feature.enabled?('geo_repository_verification')
+
+          super
+        end
+
+        private
+
+        def max_capacity
+          MAX_CAPACITY
+        end
+
+        def load_pending_resources
+          finder.find_registries_to_verify.limit(db_retrieve_batch_size).pluck(:id)
+        end
+
+        def schedule_job(registry_id)
+          job_id = Geo::RepositoryVerification::Secondary::SingleWorker.perform_async(registry_id)
+
+          { id: registry_id, job_id: job_id } if job_id
+        end
+
+        def finder
+          @finder ||= Geo::ProjectRegistryFinder.new
+        end
+      end
+    end
+  end
+end

ee/app/workers/geo/repository_verification/secondary/single_worker.rb

+module Geo
+  module RepositoryVerification
+    module Secondary
+      class SingleWorker
+        include ApplicationWorker
+        include GeoQueue
+        include ExclusiveLeaseGuard
+        include Gitlab::Geo::ProjectLogHelpers
+
+        LEASE_TIMEOUT = 1.hour.to_i
+
+        attr_reader :registry
+        private     :registry
+
+        delegate :project, to: :registry
+
+        def perform(registry_id)
+          @registry = Geo::ProjectRegistry.find(registry_id)
+          return if registry.nil? || project.pending_delete?
+
+          try_obtain_lease do
+            verify_checksum(:repository)
+            verify_checksum(:wiki)
+          end
+        end
+
+        private
+
+        def verify_checksum(type)
+          Geo::RepositoryVerifySecondaryService.new(registry, type).execute
+        rescue => e
+          log_error('Error verifying the repository checksum', e, type: type)
+          raise e
+        end
+
+        def lease_key
+          "geo:repository_verification:secondary:single_worker:#{project.id}"
+        end
+
+        def lease_timeout
+          LEASE_TIMEOUT
+        end
+      end
+    end
+  end
+end

ee/changelogs/unreleased/geo-repository-checksum.yml

+---
+title: Geo - Verify repository checksums on the secondary node
+merge_request: 4749
+author:
+type: added

ee/db/geo/migrate/20180201154345_add_repository_verification_to_project_registry.rb

+class AddRepositoryVerificationToProjectRegistry < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :project_registry, :repository_verification_checksum, :string
+    add_column :project_registry, :last_repository_verification_at, :datetime_with_timezone
+    add_column :project_registry, :last_repository_verification_failed, :boolean, null: false, default: false
+    add_column :project_registry, :last_repository_verification_failure, :string
+
+    add_column :project_registry, :wiki_verification_checksum, :string
+    add_column :project_registry, :last_wiki_verification_at, :datetime_with_timezone
+    add_column :project_registry, :last_wiki_verification_failed, :boolean, null: false, default: false
+    add_column :project_registry, :last_wiki_verification_failure, :string
+  end
+end

ee/db/geo/schema.rb

@@ -11,7 +11,7 @@
 #
 # It's strongly recommended that you check this file into your version control system.
 
-ActiveRecord::Schema.define(version: 20171115143841) do
+ActiveRecord::Schema.define(version: 20180201154345) do
 
   # These are extensions that must be enabled in order to support this database
   enable_extension "plpgsql"
@@ -52,6 +52,14 @@ ActiveRecord::Schema.define(version: 20171115143841) do
     t.boolean "force_to_redownload_wiki"
     t.string "last_repository_sync_failure"
     t.string "last_wiki_sync_failure"
+    t.string "repository_verification_checksum"
+    t.datetime_with_timezone "last_repository_verification_at"
+    t.boolean "last_repository_verification_failed", default: false, null: false
+    t.string "last_repository_verification_failure"
+    t.string "wiki_verification_checksum"
+    t.datetime_with_timezone "last_wiki_verification_at"
+    t.boolean "last_wiki_verification_failed", default: false, null: false
+    t.string "last_wiki_verification_failure"
   end
 
   add_index "project_registry", ["last_repository_successful_sync_at"], name: "index_project_registry_on_last_repository_successful_sync_at", using: :btree

ee/db/migrate/20180225180932_add_geo_node_verification_status.rb

+class AddGeoNodeVerificationStatus < ActiveRecord::Migration
+  include Gitlab::Database::MigrationHelpers
+
+  # Set this constant to true if this migration requires downtime.
+  DOWNTIME = false
+
+  def change
+    add_column :geo_node_statuses, :repositories_verified_count, :integer
+    add_column :geo_node_statuses, :repositories_verification_failed_count, :integer
+    add_column :geo_node_statuses, :wikis_verified_count, :integer
+    add_column :geo_node_statuses, :wikis_verification_failed_count, :integer
+  end
+end

ee/lib/ee/api/entities.rb

@@ -293,6 +293,18 @@ module EE
           number_to_percentage(node.wikis_synced_in_percentage, precision: 2)
         end
 
+        expose :repositories_verification_failed_count
+        expose :repositories_verified_count
+        expose :repositories_verified_in_percentage do |node|
+          number_to_percentage(node.repositories_verified_in_percentage, precision: 2)
+        end
+
+        expose :wikis_verification_failed_count
+        expose :wikis_verified_count
+        expose :wikis_verified_in_percentage do |node|
+          number_to_percentage(node.wikis_verified_in_percentage, precision: 2)
+        end
+
         expose :replication_slots_count
         expose :replication_slots_used_count
         expose :replication_slots_used_in_percentage do |node|

ee/lib/gitlab/geo/cron_manager.rb

@@ -6,6 +6,7 @@ module Gitlab
       SECONDARY_JOBS = %w[
         geo_repository_sync_worker
         geo_file_download_dispatch_worker
+        geo_repository_verification_secondary_scheduler_worker
       ].freeze
 
       GEO_JOBS = (COMMON_JOBS + PRIMARY_JOBS + SECONDARY_JOBS).freeze

ee/lib/gitlab/geo/log_helpers.rb

@@ -4,14 +4,14 @@ module Gitlab
       def log_info(message, details = {})
         data = base_log_data(message)
         data.merge!(details) if details
-        Gitlab::Geo::Logger.info(data)
+        geo_logger.info(data)
       end
 
       def log_error(message, error = nil, details = {})
         data = base_log_data(message)
         data[:error] = error.to_s if error
         data.merge!(details) if details
-        Gitlab::Geo::Logger.error(data)
+        geo_logger.error(data)
       end
 
       protected
@@ -22,6 +22,10 @@ module Gitlab
           message: message
         }
       end
+
+      def geo_logger
+        Gitlab::Geo::Logger
+      end
     end
   end
 end

ee/lib/gitlab/geo/repository_verification_log_helpers.rb

+module Gitlab
+  module Geo
+    module RepositoryVerificationLogHelpers
+      include ProjectLogHelpers
+
+      protected
+
+      def geo_logger
+        Gitlab::Geo::RepositoryVerificationLogger
+      end
+    end
+  end
+end

ee/lib/gitlab/geo/repository_verification_logger.rb

+module Gitlab
+  module Geo
+    class RepositoryVerificationLogger < ::Gitlab::Geo::Logger
+      def self.file_name_noext
+        'geo_repository_verification'
+      end
+    end
+  end
+end

ee/spec/factories/geo/project_registry.rb

@@ -72,5 +72,31 @@ FactoryBot.define do
       wiki_sync_failed
       wiki_retry_count 0
     end
+
+    trait :repository_verified do
+      repository_verification_checksum    'f079a831cab27bcda7d81cd9b48296d0c3dd92ee'
+      last_repository_verification_failed false
+      last_repository_verification_at     { 5.days.ago }
+    end
+
+    trait :wiki_verified do
+      wiki_verification_checksum    'e079a831cab27bcda7d81cd9b48296d0c3dd92ef'
+      last_wiki_verification_failed false
+      last_wiki_verification_at     { 5.days.ago }
+    end
+
+    trait :repository_verification_failed do
+      repository_verification_checksum      nil
+      last_repository_verification_at       { 5.days.ago }
+      last_repository_verification_failed   true
+      last_repository_verification_failure  'Repository checksum did not match'
+    end
+
+    trait :wiki_verification_failed do
+      wiki_verification_checksum      nil
+      last_wiki_verification_at       { 5.days.ago }
+      last_wiki_verification_failed   true
+      last_wiki_verification_failure  'Wiki checksum did not match'
+    end
   end
 end

ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json

@@ -22,6 +22,12 @@
     "wikis_count",
     "wikis_failed_count",
     "wikis_synced_count",
+    "repositories_verified_count",
+    "repositories_verification_failed_count",
+    "repositories_verified_in_percentage",
+    "wikis_verified_count",
+    "wikis_verification_failed_count",
+    "wikis_verified_in_percentage",
     "replication_slots_count",
     "replication_slots_used_count",
     "replication_slots_used_in_percentage",
@@ -64,6 +70,12 @@
     "wikis_failed_count": { "type": ["integer", "null"] },
     "wikis_synced_count": { "type": ["integer", "null"] },
     "wikis_synced_in_percentage": { "type": "string" },
+    "repositories_verified_count": { "type": ["integer", "null"] },
+    "repositories_verification_failed_count": { "type": ["integer", "null"] },
+    "repositories_verified_in_percentage": { "type": "string" },
+    "wikis_verified_count": { "type": ["integer", "null"] },
+    "wikis_verification_failed_count": { "type": ["integer", "null"] },
+    "wikis_verified_in_percentage": { "type": "string" },
     "replication_slots_count": { "type": ["integer", "null"] },
     "replication_slots_used_count": { "type": ["integer", "null"] },
     "replication_slots_used_in_percentage": { "type": "string" },

ee/spec/lib/gitlab/geo/cron_manager_spec.rb

@@ -28,6 +28,7 @@ describe Gitlab::Geo::CronManager, :geo do
       geo_repository_verification_primary_batch_worker
       geo_repository_sync_worker
       geo_file_download_dispatch_worker
+      geo_repository_verification_secondary_scheduler_worker
       geo_metrics_update_worker
     ].freeze
 
@@ -46,7 +47,8 @@ describe Gitlab::Geo::CronManager, :geo do
     let(:secondary_jobs) do
       [
         job('geo_file_download_dispatch_worker'),
-        job('geo_repository_sync_worker')
+        job('geo_repository_sync_worker'),
+        job('geo_repository_verification_secondary_scheduler_worker')
       ]
     end
 

ee/spec/models/geo/project_registry_spec.rb

@@ -45,6 +45,54 @@ describe Geo::ProjectRegistry do
     end
   end
 
+  describe '.verified_repos' do
+    it 'returns projects that verified' do
+      create(:geo_project_registry, :repository_verification_failed)
+      create(:geo_project_registry, :wiki_verified)
+      create(:geo_project_registry, :wiki_verification_failed)
+
+      repository_verified = create(:geo_project_registry, :repository_verified)
+
+      expect(described_class.verified_repos).to match_array([repository_verified])
+    end
+  end
+
+  describe '.verification_failed_repos' do
+    it 'returns projects where last attempt to verify failed' do
+      create(:geo_project_registry, :repository_verified)
+      create(:geo_project_registry, :wiki_verified)
+      create(:geo_project_registry, :wiki_verification_failed)
+
+      repository_verification_failed = create(:geo_project_registry, :repository_verification_failed)
+
+      expect(described_class.verification_failed_repos).to match_array([repository_verification_failed])
+    end
+  end
+
+  describe '.verified_wikis' do
+    it 'returns projects that verified' do
+      create(:geo_project_registry, :repository_verification_failed)
+      create(:geo_project_registry, :repository_verified)
+      create(:geo_project_registry, :wiki_verification_failed)
+
+      wiki_verified = create(:geo_project_registry, :wiki_verified)
+
+      expect(described_class.verified_wikis).to match_array([wiki_verified])
+    end
+  end
+
+  describe '.verification_failed_wikis' do
+    it 'returns projects where last attempt to verify failed' do
+      create(:geo_project_registry, :repository_verified)
+      create(:geo_project_registry, :wiki_verified)
+      create(:geo_project_registry, :repository_verification_failed)
+
+      wiki_verification_failed = create(:geo_project_registry, :wiki_verification_failed)
+
+      expect(described_class.verification_failed_wikis).to match_array([wiki_verification_failed])
+    end
+  end
+
   describe '.retry_due' do
     it 'returns projects that should be synced' do
       create(:geo_project_registry, repository_retry_at: Date.yesterday, wiki_retry_at: Date.yesterday)

ee/spec/services/geo/repository_verify_secondary_service_spec.rb

+require 'spec_helper'
+
+describe Geo::RepositoryVerifySecondaryService, :geo do
+  include ::EE::GeoHelpers
+
+  let(:primary)   { create(:geo_node, :primary) }
+  let(:secondary) { create(:geo_node) }
+
+  before do
+    stub_current_geo_node(secondary)
+  end
+
+  describe '#execute' do
+    let(:repository_state) { create(:repository_state, project: create(:project, :repository))}
+    let(:registry) do
+      registry = create(:geo_project_registry, project: repository_state.project)
+      registry.project.last_repository_updated_at = 7.hours.ago
+      registry.project.repository_state.last_repository_verification_at = 5.hours.ago
+      registry.last_repository_successful_sync_at = 5.hours.ago
+      registry.project.repository_state.repository_verification_checksum = 'my_checksum'
+
+      registry
+    end
+    let(:service)  { described_class.new(registry, :repository) }
+
+    it 'only works on the secondary' do
+      stub_current_geo_node(primary)
+
+      expect(service).not_to receive(:log_info)
+
+      service.execute
+    end
+
+    it 'sets checksum when the checksum matches' do
+      allow(service).to receive(:calculate_checksum).and_return('my_checksum')
+
+      expect(service).to receive(:record_status).once.with(checksum: 'my_checksum')
+
+      service.execute
+    end
+
+    it 'sets failure message when the checksum does not match' do
+      allow(service).to receive(:calculate_checksum).and_return('not_my_checksum')
+
+      expect(service).to receive(:record_status).once.with(error_msg: start_with('Repository checksum mismatch'))
+
+      service.execute
+    end
+  end
+
+  shared_examples 'should_verify_checksum? for repositories/wikis' do |type|
+    let(:repository_state) { create(:repository_state, project: create(:project, :repository))}
+    let(:registry) do
+      registry = create(:geo_project_registry, project: repository_state.project)
+      registry.project.last_repository_updated_at = 7.hours.ago
+      registry.project.repository_state.public_send("last_#{type}_verification_at=", 5.hours.ago)
+      registry.public_send("last_#{type}_successful_sync_at=", 5.hours.ago)
+      registry.project.repository_state.public_send("#{type}_verification_checksum=", 'my_checksum')
+
+      registry
+    end
+    let(:service)  { described_class.new(registry, type) }
+
+    it 'verifies the repository' do
+      expect(service.should_verify_checksum?).to be_truthy
+    end
+
+    it 'does not verify if primary was never verified' do
+      registry.project.repository_state.public_send("last_#{type}_verification_at=", nil)
+
+      expect(service.should_verify_checksum?).to be_falsy
+    end
+
+    it 'does not verify if the checksums already match' do
+      registry.project.repository_state.public_send("#{type}_verification_checksum=", 'my_checksum')
+      registry.public_send("#{type}_verification_checksum=", 'my_checksum')
+
+      expect(service.should_verify_checksum?).to be_falsy
+    end
+
+    it 'does not verify if the primary was verified before the secondary' do
+      registry.project.repository_state.public_send("last_#{type}_verification_at=", 50.minutes.ago)
+      registry.public_send("last_#{type}_verification_at=", 30.minutes.ago)
+
+      expect(service.should_verify_checksum?).to be_falsy
+    end
+
+    it 'does verify if the secondary was never verified' do
+      registry.public_send("last_#{type}_verification_at=", nil)
+
+      expect(service.should_verify_checksum?).to be_truthy
+    end
+
+    it 'does not verify if never synced' do
+      registry.public_send("last_#{type}_successful_sync_at=", nil)
+
+      expect(service.should_verify_checksum?).to be_falsy
+    end
+
+    it 'does not verify if the secondary synced before the last secondary verification' do
+      registry.public_send("last_#{type}_verification_at=", 50.minutes.ago)
+      registry.public_send("last_#{type}_successful_sync_at=", 30.minutes.ago)
+
+      expect(service.should_verify_checksum?).to be_falsy
+    end
+
+    it 'has been at least 6 hours since the primary repository was updated' do
+      registry.project.last_repository_updated_at = 7.hours.ago
+
+      expect(service.should_verify_checksum?).to be_truthy
+    end
+  end
+
+  describe '#should_verify_checksum?' do
+    context 'repository' do
+      include_examples 'should_verify_checksum? for repositories/wikis', :repository
+    end
+
+    context 'wiki' do
+      include_examples 'should_verify_checksum? for repositories/wikis', :wiki
+    end
+  end
+
+  shared_examples 'record_status for repositories/wikis' do |type|
+    it 'records a successful verification' do
+      service.send(:record_status, checksum: 'my_checksum')
+      registry.reload
+
+      expect(registry.public_send("#{type}_verification_checksum")).to eq 'my_checksum'
+      expect(registry.public_send("last_#{type}_verification_at")).not_to be_nil
+      expect(registry.public_send("last_#{type}_verification_failure")).to be_nil
+      expect(registry.public_send("last_#{type}_verification_failed")).to be_falsey
+    end
+
+    it 'records a failure' do
+      service.send(:record_status, error_msg: 'Repository checksum did not match')
+      registry.reload
+
+      expect(registry.public_send("#{type}_verification_checksum")).to be_nil
+      expect(registry.public_send("last_#{type}_verification_at")).not_to be_nil
+      expect(registry.public_send("last_#{type}_verification_failure")).to eq 'Repository checksum did not match'
+      expect(registry.public_send("last_#{type}_verification_failed")).to be_truthy
+    end
+  end
+
+  describe '#record_status' do
+    let(:registry) { create(:geo_project_registry) }
+
+    context 'for a repository' do
+      let(:service)  { described_class.new(registry, :repository) }
+
+      include_examples 'record_status for repositories/wikis', :repository
+    end
+
+    context 'for a wiki' do
+      let(:service)  { described_class.new(registry, :wiki) }
+
+      include_examples 'record_status for repositories/wikis', :wiki
+    end
+  end
+end

spec/javascripts/geo_nodes/mock_data.js

@@ -80,6 +80,12 @@ export const rawMockNodeDetails = {
   wikis_failed_count: 0,
   wikis_synced_count: 12,
   wikis_synced_in_percentage: '100.00%',
+  repositories_verification_failed_count: 0,
+  repositories_verified_count: 12,
+  repositories_verified_in_percentage: '100.00%',
+  wikis_verification_failed_count: 0,
+  wikis_verified_count: 12,
+  wikis_verified_in_percentage: '100.00%',
   replication_slots_count: null,
   replication_slots_used_count: null,
   replication_slots_used_in_percentage: '0.00%',