Skip to content

G
gitlab-ce
Commit 39d4c3c6 authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre
Browse files
Options

Merge branch 'mk/do-not-checksum-remote-files' into 'master' 

Geo: Exclude remote stored blobs from checksumming

See merge request gitlab-org/gitlab!54132
parents 237e3dca e56e9f51
Hide whitespace changes
Inline Side-by-side
Showing with 81 additions and 12 deletions
ee/app/assets/javascripts/geo_nodes/components/node_detail_sections/node_details_section_verification.vue
View file @ 39d4c3c6
......@@ -48,7 +48,9 @@ export default {
},
itemValue(nodeDetailItem) {
return {
totalCount: nodeDetailItem.itemValue.totalCount,
totalCount: this.nodeTypePrimary
? nodeDetailItem.itemValue.checksumTotalCount
: nodeDetailItem.itemValue.verificationTotalCount,
successCount: this.nodeTypePrimary
? nodeDetailItem.itemValue.checksumSuccessCount
: nodeDetailItem.itemValue.verificationSuccessCount,
......
ee/app/assets/javascripts/geo_nodes/store/geo_nodes_store.js
View file @ 39d4c3c6
......@@ -76,9 +76,12 @@ export default class GeoNodesStore {
totalCount: rawNodeDetails[`${replicable.namePlural}_count`],
successCount: rawNodeDetails[`${replicable.namePlural}_synced_count`],
failureCount: rawNodeDetails[`${replicable.namePlural}_failed_count`],
verificationTotalCount:
rawNodeDetails[`${replicable.namePlural}_verification_total_count`],
verificationSuccessCount: rawNodeDetails[`${replicable.namePlural}_verified_count`],
verificationFailureCount:
rawNodeDetails[`${replicable.namePlural}_verification_failed_count`],
checksumTotalCount: rawNodeDetails[`${replicable.namePlural}_checksum_total_count`],
checksumSuccessCount: rawNodeDetails[`${replicable.namePlural}_checksummed_count`],
checksumFailureCount: rawNodeDetails[`${replicable.namePlural}_checksum_failed_count`],
},
......
ee/app/models/concerns/geo/verifiable_registry.rb
View file @ 39d4c3c6
......@@ -36,6 +36,7 @@ module Geo::VerifiableRegistry
extend ::Gitlab::Utils::Override
sha_attribute :verification_checksum_mismatched
scope :available_verifiables, -> { all }
override :clear_verification_failure_fields!
def clear_verification_failure_fields!
......
ee/app/models/concerns/geo/verifiable_replicator.rb
View file @ 39d4c3c6
......@@ -128,7 +128,7 @@ module Geo
# Bonus: This causes the progress bar to be hidden.
return unless verification_enabled?
model.available_replicables.verification_succeeded.count
model.verification_succeeded.count
end
def checksum_failed_count
......@@ -136,7 +136,15 @@ module Geo
# Bonus: This causes the progress bar to be hidden.
return unless verification_enabled?
model.available_replicables.verification_failed.count
model.verification_failed.count
end
def checksum_total_count
# When verification is disabled, this returns nil.
# Bonus: This causes the progress bar to be hidden.
return unless verification_enabled?
model.available_verifiables.count
end
def verified_count
......@@ -154,6 +162,14 @@ module Geo
registry_class.synced.verification_failed.count
end
def verification_total_count
# When verification is disabled, this returns nil.
# Bonus: This causes the progress bar to be hidden.
return unless verification_enabled?
registry_class.synced.available_verifiables.count
end
end
def handle_after_checksum_succeeded
......
ee/app/models/geo_node_status.rb
View file @ 39d4c3c6
......@@ -34,11 +34,13 @@ class GeoNodeStatus < ApplicationRecord
def self.status_fields_for(replicable_class)
{
"#{replicable_class.replicable_name_plural}_count".to_sym => "Number of #{replicable_class.replicable_title_plural} on the primary",
"#{replicable_class.replicable_name_plural}_checksum_total_count".to_sym => "Number of #{replicable_class.replicable_title_plural} available to checksum on primary",
"#{replicable_class.replicable_name_plural}_checksummed_count".to_sym => "Number of #{replicable_class.replicable_title_plural} checksummed on the primary",
"#{replicable_class.replicable_name_plural}_checksum_failed_count".to_sym => "Number of #{replicable_class.replicable_title_plural} failed to checksum on primary",
"#{replicable_class.replicable_name_plural}_synced_count".to_sym => "Number of #{replicable_class.replicable_title_plural} in the registry",
"#{replicable_class.replicable_name_plural}_failed_count".to_sym => "Number of #{replicable_class.replicable_title_plural} synced on secondary",
"#{replicable_class.replicable_name_plural}_registry_count".to_sym => "Number of #{replicable_class.replicable_title_plural} failed to sync on secondary",
"#{replicable_class.replicable_name_plural}_verification_total_count".to_sym => "Number of #{replicable_class.replicable_title_plural} available to verify on secondary",
"#{replicable_class.replicable_name_plural}_verified_count".to_sym => "Number of #{replicable_class.replicable_title_plural} verified on the secondary",
"#{replicable_class.replicable_name_plural}_verification_failed_count".to_sym => "Number of #{replicable_class.replicable_title_plural} failed to verify on secondary"
}
......@@ -72,17 +74,21 @@ class GeoNodeStatus < ApplicationRecord
job_artifacts_failed_count
repositories_verified_count
repositories_verification_failed_count
repositories_verification_total_count
wikis_verified_count
wikis_verification_failed_count
wikis_verification_total_count
lfs_objects_synced_missing_on_primary_count
job_artifacts_synced_missing_on_primary_count
attachments_synced_missing_on_primary_count
repositories_checksummed_count
repositories_checksum_failed_count
repositories_checksum_mismatch_count
repositories_checksum_total_count
wikis_checksummed_count
wikis_checksum_failed_count
wikis_checksum_mismatch_count
wikis_checksum_total_count
repositories_retrying_verification_count
wikis_retrying_verification_count
projects_count
......@@ -562,21 +568,26 @@ class GeoNodeStatus < ApplicationRecord
def load_primary_verification_data
self.repositories_checksummed_count = repository_verification_finder.count_verified_repositories
self.repositories_checksum_failed_count = repository_verification_finder.count_verification_failed_repositories
self.repositories_checksum_total_count = self.projects_count
self.wikis_checksummed_count = repository_verification_finder.count_verified_wikis
self.wikis_checksum_failed_count = repository_verification_finder.count_verification_failed_wikis
self.wikis_checksum_total_count = self.projects_count
Gitlab::Geo::REPLICATOR_CLASSES.each do |replicator|
public_send("#{replicator.replicable_name_plural}_checksummed_count=", replicator.checksummed_count) # rubocop:disable GitlabSecurity/PublicSend
public_send("#{replicator.replicable_name_plural}_checksum_failed_count=", replicator.checksum_failed_count) # rubocop:disable GitlabSecurity/PublicSend
public_send("#{replicator.replicable_name_plural}_checksum_total_count=", replicator.checksum_total_count) # rubocop:disable GitlabSecurity/PublicSend
end
end
def load_secondary_verification_data
self.repositories_verified_count = Geo::ProjectRegistry.verified(:repository).count
self.repositories_verification_failed_count = Geo::ProjectRegistry.verification_failed(:repository).count
self.repositories_verification_total_count = self.projects_count
self.repositories_checksum_mismatch_count = Geo::ProjectRegistry.mismatch(:repository).count
self.wikis_verified_count = Geo::ProjectRegistry.verified(:wiki).count
self.wikis_verification_failed_count = Geo::ProjectRegistry.verification_failed(:wiki).count
self.wikis_verification_total_count = self.projects_count
self.wikis_checksum_mismatch_count = Geo::ProjectRegistry.mismatch(:wiki).count
self.repositories_retrying_verification_count = Geo::ProjectRegistry.retrying_verification(:repository).count
self.wikis_retrying_verification_count = Geo::ProjectRegistry.retrying_verification(:wiki).count
......@@ -584,6 +595,7 @@ class GeoNodeStatus < ApplicationRecord
::Gitlab::Geo.verification_enabled_replicator_classes.each do |replicator|
public_send("#{replicator.replicable_name_plural}_verified_count=", replicator.verified_count) # rubocop:disable GitlabSecurity/PublicSend
public_send("#{replicator.replicable_name_plural}_verification_failed_count=", replicator.verification_failed_count) # rubocop:disable GitlabSecurity/PublicSend
public_send("#{replicator.replicable_name_plural}_verification_total_count=", replicator.verification_total_count) # rubocop:disable GitlabSecurity/PublicSend
end
end
......
ee/lib/gitlab/geo/replicable_model.rb
View file @ 39d4c3c6
......@@ -18,7 +18,10 @@ module Gitlab
# all models. https://gitlab.com/gitlab-org/gitlab/-/issues/280768
scope :verification_succeeded, -> { none }
scope :verification_failed, -> { none }
# These scopes are intended to be overridden as needed
scope :available_replicables, -> { all }
scope :available_verifiables, -> { self.respond_to?(:with_files_stored_locally) ? available_replicables.with_files_stored_locally : available_replicables }
end
class_methods do
......
ee/lib/gitlab/geo/verification_state.rb
View file @ 39d4c3c6
......@@ -28,15 +28,15 @@ module Gitlab
sha_attribute :verification_checksum
# rubocop:disable CodeReuse/ActiveRecord
scope :verification_pending, -> { with_verification_state(:verification_pending) }
scope :verification_started, -> { with_verification_state(:verification_started) }
scope :verification_succeeded, -> { with_verification_state(:verification_succeeded) }
scope :verification_failed, -> { with_verification_state(:verification_failed) }
scope :verification_pending, -> { available_verifiables.with_verification_state(:verification_pending) }
scope :verification_started, -> { available_verifiables.with_verification_state(:verification_started) }
scope :verification_succeeded, -> { available_verifiables.with_verification_state(:verification_succeeded) }
scope :verification_failed, -> { available_verifiables.with_verification_state(:verification_failed) }
scope :checksummed, -> { where.not(verification_checksum: nil) }
scope :not_checksummed, -> { where(verification_checksum: nil) }
scope :verification_timed_out, -> { verification_started.where("verification_started_at < ?", VERIFICATION_TIMEOUT.ago) }
scope :retry_due, -> { where(arel_table[:verification_retry_at].eq(nil).or(arel_table[:verification_retry_at].lt(Time.current))) }
scope :needs_verification, -> { with_verification_state(:verification_pending).or(with_verification_state(:verification_failed).retry_due) }
scope :needs_verification, -> { available_verifiables.merge(with_verification_state(:verification_pending).or(with_verification_state(:verification_failed).retry_due)) }
scope :needs_reverification, -> { verification_succeeded.where("verified_at < ?", ::Gitlab::Geo.current_node.minimum_reverification_interval.days.ago) }
# rubocop:enable CodeReuse/ActiveRecord
......
ee/spec/factories/geo_node_statuses.rb
View file @ 39d4c3c6
......@@ -35,13 +35,17 @@ FactoryBot.define do
wikis_failed_count { 1 }
repositories_checksummed_count { 600 }
repositories_checksum_failed_count { 120 }
repositories_checksum_total_count { 120 }
wikis_checksummed_count { 585 }
wikis_checksum_failed_count { 55 }
wikis_checksum_total_count { 55 }
repositories_verified_count { 501 }
repositories_verification_failed_count { 100 }
repositories_verification_total_count { 100 }
repositories_checksum_mismatch_count { 15 }
wikis_verified_count { 499 }
wikis_verification_failed_count { 99 }
wikis_verification_total_count { 99 }
wikis_checksum_mismatch_count { 10 }
repositories_retrying_verification_count { 25 }
wikis_retrying_verification_count { 3 }
......@@ -80,11 +84,15 @@ FactoryBot.define do
repositories_synced_count { 10 }
repositories_checksummed_count { 10 }
repositories_checksum_total_count { 10 }
repositories_verified_count { 10 }
repositories_verification_total_count { 10 }
repositories_checked_count { 10 }
wikis_synced_count { 10 }
wikis_checksummed_count { 10 }
wikis_checksum_total_count { 10 }
wikis_verified_count { 10 }
wikis_verification_total_count { 10 }
lfs_objects_synced_count { 10 }
job_artifacts_synced_count { 10 }
attachments_synced_count { 10 }
......
ee/spec/fixtures/api/schemas/public_api/v4/geo_node_status.json
View file @ 39d4c3c6
......@@ -42,56 +42,68 @@
"repository_verification_enabled",
"repositories_checksummed_count",
"repositories_checksum_failed_count",
"repositories_checksum_total_count",
"repositories_checksummed_in_percentage",
"wikis_checksummed_count",
"wikis_checksum_failed_count",
"wikis_checksum_total_count",
"wikis_checksummed_in_percentage",
"merge_request_diffs_count",
"merge_request_diffs_checksum_failed_count",
"merge_request_diffs_checksum_total_count",
"merge_request_diffs_checksummed_count",
"merge_request_diffs_registry_count",
"merge_request_diffs_failed_count",
"merge_request_diffs_synced_count",
"merge_request_diffs_synced_in_percentage",
"merge_request_diffs_verification_failed_count",
"merge_request_diffs_verification_total_count",
"merge_request_diffs_verified_count",
"merge_request_diffs_verified_in_percentage",
"package_files_count",
"package_files_checksum_failed_count",
"package_files_checksum_total_count",
"package_files_checksummed_count",
"package_files_registry_count",
"package_files_failed_count",
"package_files_synced_count",
"package_files_synced_in_percentage",
"package_files_verification_failed_count",
"package_files_verification_total_count",
"package_files_verified_count",
"package_files_verified_in_percentage",
"terraform_state_versions_count",
"terraform_state_versions_checksum_failed_count",
"terraform_state_versions_checksum_total_count",
"terraform_state_versions_checksummed_count",
"terraform_state_versions_registry_count",
"terraform_state_versions_failed_count",
"terraform_state_versions_synced_count",
"terraform_state_versions_synced_in_percentage",
"terraform_state_versions_verification_failed_count",
"terraform_state_versions_verification_total_count",
"terraform_state_versions_verified_count",
"terraform_state_versions_verified_in_percentage",
"snippet_repositories_count",
"snippet_repositories_checksum_failed_count",
"snippet_repositories_checksum_total_count",
"snippet_repositories_checksummed_count",
"snippet_repositories_registry_count",
"snippet_repositories_failed_count",
"snippet_repositories_synced_count",
"snippet_repositories_synced_in_percentage",
"snippet_repositories_verification_failed_count",
"snippet_repositories_verification_total_count",
"snippet_repositories_verified_count",
"snippet_repositories_verified_in_percentage",
"repositories_verified_count",
"repositories_verification_failed_count",
"repositories_verification_total_count",
"repositories_verified_in_percentage",
"repositories_checksum_mismatch_count",
"wikis_verified_count",
"wikis_verification_failed_count",
"wikis_verification_total_count",
"wikis_verified_in_percentage",
"wikis_checksum_mismatch_count",
"repositories_retrying_verification_count",
......@@ -163,56 +175,68 @@
"wikis_synced_in_percentage": { "type": "string" },
"repositories_checksummed_count": { "type": ["integer", "null"] },
"repositories_checksum_failed_count": { "type": ["integer", "null"] },
"repositories_checksum_total_count": { "type": ["integer", "null"] },
"repositories_checksummed_in_percentage": { "type": "string" },
"wikis_checksummed_count": { "type": ["integer", "null"] },
"wikis_checksum_failed_count": { "type": ["integer", "null"] },
"wikis_checksum_total_count": { "type": ["integer", "null"] },
"wikis_checksummed_in_percentage": { "type": "string" },
"merge_request_diffs_count": { "type": ["integer", "null"] },
"merge_request_diffs_checksummed_count": { "type": ["integer", "null"] },
"merge_request_diffs_checksum_failed_count": { "type": ["integer", "null"] },
"merge_request_diffs_checksum_total_count": { "type": ["integer", "null"] },
"merge_request_diffs_registry_count": { "type": ["integer", "null"] },
"merge_request_diffs_failed_count": { "type": ["integer", "null"] },
"merge_request_diffs_synced_count": { "type": ["integer", "null"] },
"merge_request_diffs_synced_in_percentage": { "type": "string" },
"merge_request_diffs_verification_failed_count": { "type": ["integer", "null"] },
"merge_request_diffs_verification_total_count": { "type": ["integer", "null"] },
"merge_request_diffs_verified_count": { "type": ["integer", "null"] },
"merge_request_diffs_verified_in_percentage": { "type": "string" },
"package_files_count": { "type": ["integer", "null"] },
"package_files_checksummed_count": { "type": ["integer", "null"] },
"package_files_checksum_failed_count": { "type": ["integer", "null"] },
"package_files_checksum_total_count": { "type": ["integer", "null"] },
"package_files_registry_count": { "type": ["integer", "null"] },
"package_files_failed_count": { "type": ["integer", "null"] },
"package_files_synced_count": { "type": ["integer", "null"] },
"package_files_synced_in_percentage": { "type": "string" },
"package_files_verification_failed_count": { "type": ["integer", "null"] },
"package_files_verification_total_count": { "type": ["integer", "null"] },
"package_files_verified_count": { "type": ["integer", "null"] },
"package_files_verified_in_percentage": { "type": "string" },
"terraform_state_versions_count": { "type": ["integer", "null"] },
"terraform_state_versions_checksummed_count": { "type": ["integer", "null"] },
"terraform_state_versions_checksum_failed_count": { "type": ["integer", "null"] },
"terraform_state_versions_checksum_total_count": { "type": ["integer", "null"] },
"terraform_state_versions_registry_count": { "type": ["integer", "null"] },
"terraform_state_versions_failed_count": { "type": ["integer", "null"] },
"terraform_state_versions_synced_count": { "type": ["integer", "null"] },
"terraform_state_versions_synced_in_percentage": { "type": "string" },
"terraform_state_versions_verification_failed_count": { "type": ["integer", "null"] },
"terraform_state_versions_verification_total_count": { "type": ["integer", "null"] },
"terraform_state_versions_verified_count": { "type": ["integer", "null"] },
"terraform_state_versions_verified_in_percentage": { "type": "string" },
"snippet_repositories_count": { "type": ["integer", "null"] },
"snippet_repositories_checksummed_count": { "type": ["integer", "null"] },
"snippet_repositories_checksum_failed_count": { "type": ["integer", "null"] },
"snippet_repositories_checksum_total_count": { "type": ["integer", "null"] },
"snippet_repositories_registry_count": { "type": ["integer", "null"] },
"snippet_repositories_failed_count": { "type": ["integer", "null"] },
"snippet_repositories_synced_count": { "type": ["integer", "null"] },
"snippet_repositories_synced_in_percentage": { "type": "string" },
"snippet_repositories_verification_failed_count": { "type": ["integer", "null"] },
"snippet_repositories_verification_total_count": { "type": ["integer", "null"] },
"snippet_repositories_verified_count": { "type": ["integer", "null"] },
"snippet_repositories_verified_in_percentage": { "type": "string" },
"repositories_verified_count": { "type": ["integer", "null"] },
"repositories_verification_failed_count": { "type": ["integer", "null"] },
"repositories_verification_total_count": { "type": ["integer", "null"] },
"repositories_verified_in_percentage": { "type": "string" },
"repositories_checksum_mismatch_count": { "type": ["integer", "null"] },
"wikis_verified_count": { "type": ["integer", "null"] },
"wikis_verification_failed_count": { "type": ["integer", "null"] },
"wikis_verification_total_count": { "type": ["integer", "null"] },
"wikis_verified_in_percentage": { "type": "string" },
"wikis_checksum_mismatch_count": { "type": ["integer", "null"] },
"repositories_retrying_verification_count": { "type": ["integer", "null"] },
......
ee/spec/support/shared_examples/models/concerns/verifiable_replicator_shared_examples.rb
View file @ 39d4c3c6
......@@ -58,14 +58,14 @@ RSpec.shared_examples 'a verifiable replicator' do
allow(described_class).to receive(:verification_enabled?).and_return(true)
end
it 'returns the number of available replicables where verification succeeded' do
model_record.verification_started!
it 'returns the number of available verifiables where verification succeeded' do
model_record.verification_started
model_record.verification_succeeded_with_checksum!('some checksum', Time.current)
expect(described_class.checksummed_count).to eq(1)
end
it 'excludes other verification states' do
it 'excludes non-success verification states' do
model_record.verification_started!
expect(described_class.checksummed_count).to eq(0)
......@@ -95,7 +95,7 @@ RSpec.shared_examples 'a verifiable replicator' do
allow(described_class).to receive(:verification_enabled?).and_return(true)
end
it 'returns the number of available replicables where verification failed' do
it 'returns the number of available verifiables where verification failed' do
model_record.verification_started!
model_record.verification_failed_with_message!('some error message')
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7