Skip to content

G
gitlab-ce
Commit cc088fff authored by Nick Thomas's avatar Nick Thomas
Browse files
Options

Merge branch... 

Merge branch '11001-geo-implement-selective-sync-support-for-the-job-artifacts-fdw-queries' into 'master'

Geo - Add selective sync support for the job artifacts FDW queries

Closes #11001

See merge request gitlab-org/gitlab-ee!11892
parents e1c4d28e 79822cfa
Hide whitespace changes
Inline Side-by-side
Showing with 573 additions and 466 deletions
ee/app/finders/geo/job_artifact_registry_finder.rb
View file @ cc088fff
...@@ -7,31 +7,29 @@ module Geo ...@@ -7,31 +7,29 @@ module Geo
end end
def count_synced def count_synced
if aggregate_pushdown_supported? job_artifacts_synced.count
find_synced.count
else
legacy_find_synced.count
end
end end
def count_failed def count_failed
if aggregate_pushdown_supported? job_artifacts_failed.count
find_failed.count
else
legacy_find_failed.count
end
end end
def count_synced_missing_on_primary def count_synced_missing_on_primary
if aggregate_pushdown_supported? job_artifacts_synced_missing_on_primary.count
find_synced_missing_on_primary.count
else
legacy_find_synced_missing_on_primary.count
end
end end
def count_registry def count_registry
Geo::JobArtifactRegistry.count registries_for_job_artifacts.count
end
def syncable
if use_legacy_queries_for_selective_sync?
legacy_finder.syncable
elsif selective_sync?
fdw_geo_node.job_artifacts.syncable
else
Ci::JobArtifact.syncable
end
end end
# Find limited amount of non replicated job artifacts. # Find limited amount of non replicated job artifacts.
...@@ -47,10 +45,10 @@ module Geo ...@@ -47,10 +45,10 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_artifact_ids: []) def find_unsynced(batch_size:, except_artifact_ids: [])
relation = relation =
if use_legacy_queries? if use_legacy_queries_for_selective_sync?
legacy_find_unsynced(except_artifact_ids: except_artifact_ids) legacy_finder.job_artifacts_unsynced(except_artifact_ids: except_artifact_ids)
else else
fdw_find_unsynced(except_artifact_ids: except_artifact_ids) job_artifacts_unsynced(except_artifact_ids: except_artifact_ids)
end end
relation.limit(batch_size) relation.limit(batch_size)
...@@ -60,186 +58,98 @@ module Geo ...@@ -60,186 +58,98 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local(batch_size:, except_artifact_ids: []) def find_migrated_local(batch_size:, except_artifact_ids: [])
relation = relation =
if use_legacy_queries? if use_legacy_queries_for_selective_sync?
legacy_find_migrated_local(except_artifact_ids: except_artifact_ids) legacy_finder.job_artifacts_migrated_local(except_artifact_ids: except_artifact_ids)
else else
fdw_find_migrated_local(except_artifact_ids: except_artifact_ids) job_artifacts_migrated_local(except_artifact_ids: except_artifact_ids)
end end
relation.limit(batch_size) relation.limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
def syncable
all.geo_syncable
end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_artifact_ids: []) def find_retryable_failed_registries(batch_size:, except_artifact_ids: [])
find_failed_registries Geo::JobArtifactRegistry
.failed
.retry_due .retry_due
.where.not(artifact_id: except_artifact_ids) .artifact_id_not_in(except_artifact_ids)
.limit(batch_size) .limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_artifact_ids: []) def find_retryable_synced_missing_on_primary_registries(batch_size:, except_artifact_ids: [])
find_synced_missing_on_primary_registries Geo::JobArtifactRegistry
.synced
.missing_on_primary
.retry_due .retry_due
.where.not(artifact_id: except_artifact_ids) .artifact_id_not_in(except_artifact_ids)
.limit(batch_size) .limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
private private
# rubocop: disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/Finder
def all def legacy_finder
if selective_sync? @legacy_finder ||= Geo::LegacyJobArtifactRegistryFinder.new(current_node: current_node)
Ci::JobArtifact.joins(:project).where(projects: { id: current_node.projects })
else
Ci::JobArtifact.all
end
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/Finder
def find_synced def fdw_geo_node
if use_legacy_queries? @fdw_geo_node ||= Geo::Fdw::GeoNode.find(current_node.id)
legacy_find_synced
else
fdw_find.merge(find_synced_registries)
end
end end
def find_synced_missing_on_primary def registries_for_job_artifacts
if use_legacy_queries? if use_legacy_queries_for_selective_sync?
legacy_find_synced_missing_on_primary legacy_finder.registries_for_job_artifacts
else else
fdw_find.merge(find_synced_missing_on_primary_registries) fdw_geo_node
.job_artifacts
.inner_join_job_artifact_registry
.syncable
end end
end end
def find_failed def job_artifacts_synced
if use_legacy_queries? if use_legacy_queries_for_selective_sync?
legacy_find_failed legacy_finder.job_artifacts_synced
else else
fdw_find.merge(find_failed_registries) registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.synced)
end end
end end
def find_synced_registries def job_artifacts_failed
Geo::JobArtifactRegistry.synced if use_legacy_queries_for_selective_sync?
end legacy_finder.job_artifacts_failed
def find_synced_missing_on_primary_registries
find_synced_registries.missing_on_primary
end
def find_failed_registries
Geo::JobArtifactRegistry.failed
end
#
# FDW accessors
#
# rubocop: disable CodeReuse/ActiveRecord
def fdw_find
fdw_all.joins("INNER JOIN job_artifact_registry ON job_artifact_registry.artifact_id = #{fdw_table}.id")
.geo_syncable
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def fdw_find_unsynced(except_artifact_ids:)
fdw_all.joins("LEFT OUTER JOIN job_artifact_registry
ON job_artifact_registry.artifact_id = #{fdw_table}.id")
.geo_syncable
.where(job_artifact_registry: { artifact_id: nil })
.where.not(id: except_artifact_ids)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def fdw_find_migrated_local(except_artifact_ids:)
fdw_all.joins("INNER JOIN job_artifact_registry ON job_artifact_registry.artifact_id = #{fdw_table}.id")
.with_files_stored_remotely
.where.not(id: except_artifact_ids)
.merge(Geo::JobArtifactRegistry.all)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def fdw_all
if selective_sync?
Geo::Fdw::Ci::JobArtifact.joins(:project).where(projects: { id: current_node.projects })
else else
Geo::Fdw::Ci::JobArtifact.all registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.failed)
end end
end end
# rubocop: enable CodeReuse/ActiveRecord
def fdw_table
Geo::Fdw::Ci::JobArtifact.table_name
end
#
# Legacy accessors (non FDW)
#
# rubocop: disable CodeReuse/ActiveRecord
def legacy_find_synced
legacy_inner_join_registry_ids(
syncable,
find_synced_registries.pluck(:artifact_id),
Ci::JobArtifact
)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord def job_artifacts_synced_missing_on_primary
def legacy_find_failed if use_legacy_queries_for_selective_sync?
legacy_inner_join_registry_ids( legacy_finder.job_artifacts_synced_missing_on_primary
syncable, else
find_failed_registries.pluck(:artifact_id), registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.synced.missing_on_primary)
Ci::JobArtifact end
)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def legacy_find_unsynced(except_artifact_ids:)
registry_artifact_ids = Geo::JobArtifactRegistry.pluck(:artifact_id) | except_artifact_ids
legacy_left_outer_join_registry_ids(
syncable,
registry_artifact_ids,
Ci::JobArtifact
)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def legacy_find_migrated_local(except_artifact_ids:)
registry_artifact_ids = Geo::JobArtifactRegistry.pluck(:artifact_id) - except_artifact_ids
legacy_inner_join_registry_ids( def job_artifacts_unsynced(except_artifact_ids:)
all.with_files_stored_remotely, fdw_geo_node
registry_artifact_ids, .job_artifacts
Ci::JobArtifact .syncable
) .missing_job_artifact_registry
.id_not_in(except_artifact_ids)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord def job_artifacts_migrated_local(except_artifact_ids:)
def legacy_find_synced_missing_on_primary fdw_geo_node
legacy_inner_join_registry_ids( .job_artifacts
syncable, .inner_join_job_artifact_registry
find_synced_missing_on_primary_registries.pluck(:artifact_id), .with_files_stored_remotely
Ci::JobArtifact .id_not_in(except_artifact_ids)
)
end end
# rubocop: enable CodeReuse/ActiveRecord
end end
end end
ee/app/finders/geo/legacy_job_artifact_registry_finder.rb 0 → 100644
View file @ cc088fff
# frozen_string_literal: true
module Geo
class LegacyJobArtifactRegistryFinder < RegistryFinder
def syncable
current_node.job_artifacts.syncable
end
def job_artifacts_synced
legacy_inner_join_registry_ids(
syncable,
Geo::JobArtifactRegistry.synced.pluck_artifact_key,
Ci::JobArtifact
)
end
def job_artifacts_failed
legacy_inner_join_registry_ids(
syncable,
Geo::JobArtifactRegistry.failed.pluck_artifact_key,
Ci::JobArtifact
)
end
def job_artifacts_synced_missing_on_primary
legacy_inner_join_registry_ids(
syncable,
Geo::JobArtifactRegistry.synced.missing_on_primary.pluck_artifact_key,
Ci::JobArtifact
)
end
def job_artifacts_unsynced(except_artifact_ids: [])
registry_artifact_ids = Geo::JobArtifactRegistry.pluck_artifact_key | except_artifact_ids
legacy_left_outer_join_registry_ids(
syncable,
registry_artifact_ids,
Ci::JobArtifact
)
end
def job_artifacts_migrated_local(except_artifact_ids: [])
registry_artifact_ids = Geo::JobArtifactRegistry.pluck_artifact_key - except_artifact_ids
legacy_inner_join_registry_ids(
current_node.job_artifacts.with_files_stored_remotely,
registry_artifact_ids,
Ci::JobArtifact
)
end
def registries_for_job_artifacts
return Geo::JobArtifactRegistry.all unless selective_sync?
legacy_inner_join_registry_ids(
Geo::JobArtifactRegistry.all,
current_node.job_artifacts.pluck_primary_key,
Geo::JobArtifactRegistry,
foreign_key: :artifact_id
)
end
end
end
ee/app/models/ee/ci/job_artifact.rb
View file @ cc088fff
...@@ -16,7 +16,8 @@ module EE ...@@ -16,7 +16,8 @@ module EE
METRICS_REPORT_FILE_TYPES = %w[metrics].freeze METRICS_REPORT_FILE_TYPES = %w[metrics].freeze
scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
scope :geo_syncable, -> { with_files_stored_locally.not_expired } scope :project_id_in, ->(ids) { joins(:project).merge(::Project.id_in(ids)) }
scope :syncable, -> { with_files_stored_locally.not_expired }
scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
scope :security_reports, -> do scope :security_reports, -> do
......
ee/app/models/geo/fdw/ci/job_artifact.rb
View file @ cc088fff
...@@ -10,8 +10,42 @@ module Geo ...@@ -10,8 +10,42 @@ module Geo
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('ci_job_artifacts') self.table_name = Gitlab::Geo::Fdw.foreign_table_name('ci_job_artifacts')
belongs_to :project, class_name: 'Geo::Fdw::Project', inverse_of: :job_artifacts
scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
scope :geo_syncable, -> { with_files_stored_locally.not_expired } scope :project_id_in, ->(ids) { joins(:project).merge(Geo::Fdw::Project.id_in(ids)) }
scope :syncable, -> { with_files_stored_locally.not_expired }
class << self
def inner_join_job_artifact_registry
join_statement =
arel_table
.join(job_artifact_registry_table, Arel::Nodes::InnerJoin)
.on(arel_table[:id].eq(job_artifact_registry_table[:artifact_id]))
joins(join_statement.join_sources)
end
def missing_job_artifact_registry
left_outer_join_job_artifact_registry
.where(job_artifact_registry_table[:id].eq(nil))
end
private
def job_artifact_registry_table
Geo::JobArtifactRegistry.arel_table
end
def left_outer_join_job_artifact_registry
join_statement =
arel_table
.join(job_artifact_registry_table, Arel::Nodes::OuterJoin)
.on(arel_table[:id].eq(job_artifact_registry_table[:artifact_id]))
joins(join_statement.join_sources)
end
end
end end
end end
end end
......
ee/app/models/geo/fdw/geo_node.rb
View file @ cc088fff
...@@ -14,6 +14,12 @@ module Geo ...@@ -14,6 +14,12 @@ module Geo
has_many :geo_node_namespace_links, class_name: 'Geo::Fdw::GeoNodeNamespaceLink' has_many :geo_node_namespace_links, class_name: 'Geo::Fdw::GeoNodeNamespaceLink'
has_many :namespaces, class_name: 'Geo::Fdw::Namespace', through: :geo_node_namespace_links has_many :namespaces, class_name: 'Geo::Fdw::Namespace', through: :geo_node_namespace_links
def job_artifacts
Geo::Fdw::Ci::JobArtifact.all unless selective_sync?
Geo::Fdw::Ci::JobArtifact.project_id_in(projects)
end
def lfs_objects def lfs_objects
return Geo::Fdw::LfsObject.all unless selective_sync? return Geo::Fdw::LfsObject.all unless selective_sync?
......
ee/app/models/geo/fdw/project.rb
View file @ cc088fff
...@@ -7,6 +7,7 @@ module Geo ...@@ -7,6 +7,7 @@ module Geo
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('projects') self.table_name = Gitlab::Geo::Fdw.foreign_table_name('projects')
has_many :job_artifacts, class_name: 'Geo::Fdw::Ci::JobArtifact'
has_many :lfs_objects_projects, class_name: 'Geo::Fdw::LfsObjectsProject' has_many :lfs_objects_projects, class_name: 'Geo::Fdw::LfsObjectsProject'
has_many :lfs_objects, class_name: 'Geo::Fdw::LfsObject', through: :lfs_objects_projects has_many :lfs_objects, class_name: 'Geo::Fdw::LfsObject', through: :lfs_objects_projects
......
ee/app/models/geo/job_artifact_registry.rb
View file @ cc088fff
...@@ -2,4 +2,12 @@ ...@@ -2,4 +2,12 @@
class Geo::JobArtifactRegistry < Geo::BaseRegistry class Geo::JobArtifactRegistry < Geo::BaseRegistry
include Geo::Syncable include Geo::Syncable
def self.artifact_id_not_in(ids)
where.not(artifact_id: ids)
end
def self.pluck_artifact_key
where(nil).pluck(:artifact_id)
end
end end
ee/app/models/geo_node.rb
View file @ cc088fff
...@@ -198,6 +198,12 @@ class GeoNode < ApplicationRecord ...@@ -198,6 +198,12 @@ class GeoNode < ApplicationRecord
end end
end end
def job_artifacts
Ci::JobArtifact.all unless selective_sync?
Ci::JobArtifact.project_id_in(projects)
end
def lfs_objects def lfs_objects
return LfsObject.all unless selective_sync? return LfsObject.all unless selective_sync?
......
ee/changelogs/unreleased/11001-geo-implement-selective-sync-support-for-the-job-artifacts-fdw-queries.yml 0 → 100644
View file @ cc088fff
---
title: Geo - Add selective sync support for the job artifacts FDW queries
merge_request: 11892
author:
type: changed
ee/spec/finders/geo/attachment_registry_finder_spec.rb
View file @ cc088fff
...@@ -28,25 +28,6 @@ describe Geo::AttachmentRegistryFinder, :geo do ...@@ -28,25 +28,6 @@ describe Geo::AttachmentRegistryFinder, :geo do
stub_current_geo_node(secondary) stub_current_geo_node(secondary)
end end
it 'responds to file registry finder methods' do
file_registry_finder_methods = %i{
syncable
count_syncable
count_synced
count_failed
count_synced_missing_on_primary
count_registry
find_unsynced
find_migrated_local
find_retryable_failed_registries
find_retryable_synced_missing_on_primary_registries
}
file_registry_finder_methods.each do |method|
expect(subject).to respond_to(method)
end
end
shared_examples 'finds all the things' do shared_examples 'finds all the things' do
describe '#find_unsynced' do describe '#find_unsynced' do
let!(:upload_1) { create(:upload, model: synced_group) } let!(:upload_1) { create(:upload, model: synced_group) }
...@@ -475,32 +456,5 @@ describe Geo::AttachmentRegistryFinder, :geo do ...@@ -475,32 +456,5 @@ describe Geo::AttachmentRegistryFinder, :geo do
end end
end end
context 'FDW', :geo_fdw do it_behaves_like 'a file registry finder'
context 'with use_fdw_queries_for_selective_sync disabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: false)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
context 'with use_fdw_queries_for_selective_sync enabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: true)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
end
context 'Legacy' do
before do
stub_fdw_disabled
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
end end
ee/spec/finders/geo/job_artifact_registry_finder_spec.rb
View file @ cc088fff
...@@ -11,13 +11,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -11,13 +11,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
let(:synced_group) { create(:group) } let(:synced_group) { create(:group) }
let(:synced_project) { create(:project, group: synced_group) } let(:synced_project) { create(:project, group: synced_group) }
let(:unsynced_project) { create(:project) } let(:unsynced_project) { create(:project) }
let(:project_broken_storage) { create(:project, :broken_storage) }
let(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let(:job_artifact_remote_1) { create(:ci_job_artifact, :remote_store, project: synced_project) }
let(:job_artifact_remote_2) { create(:ci_job_artifact, :remote_store, project: unsynced_project) }
subject { described_class.new(current_node: secondary) } subject { described_class.new(current_node: secondary) }
...@@ -28,30 +22,30 @@ describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -28,30 +22,30 @@ describe Geo::JobArtifactRegistryFinder, :geo do
shared_examples 'counts all the things' do shared_examples 'counts all the things' do
describe '#count_syncable' do describe '#count_syncable' do
before do let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
job_artifact_1 let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
job_artifact_2 let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
job_artifact_3 let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
job_artifact_4 let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
end let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
it 'counts job artifacts' do it 'counts job artifacts' do
expect(subject.count_syncable).to eq 4 expect(subject.count_syncable).to eq 6
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE) job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_syncable).to eq 3 expect(subject.count_syncable).to eq 5
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_syncable).to eq 3 expect(subject.count_syncable).to eq 5
end end
context 'with selective sync' do context 'with selective sync by namespace' do
before do before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end end
...@@ -72,168 +66,198 @@ describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -72,168 +66,198 @@ describe Geo::JobArtifactRegistryFinder, :geo do
expect(subject.count_syncable).to eq 1 expect(subject.count_syncable).to eq 1
end end
end end
end
describe '#count_synced' do context 'with selective sync by shard' do
it 'delegates to #legacy_find_synced' do before do
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(false) secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
expect(subject).to receive(:legacy_find_synced).and_call_original it 'counts job artifacts' do
expect(subject.count_syncable).to eq 2
end
subject.count_synced it 'ignores remote job artifacts' do
end job_artifact_5.update_column(:file_store, ObjectStorage::Store::REMOTE)
it 'delegates to #find_synced for PostgreSQL 10' do expect(subject.count_syncable).to eq 1
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(true) end
expect(subject).to receive(:find_synced).and_call_original it 'ignores expired job artifacts' do
job_artifact_5.update_column(:expire_at, Date.yesterday)
subject.count_synced expect(subject.count_syncable).to eq 1
end
end end
end
it 'counts job artifacts that have been synced' do describe '#count_synced' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id) let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id) let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
end
expect(subject.count_synced).to eq 2 it 'counts job artifacts that have been synced' do
end expect(subject.count_synced).to eq 2
end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id) job_artifact_2.update_column(:file_store, ObjectStorage::Store::REMOTE)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
expect(subject.count_synced).to eq 2 expect(subject.count_synced).to eq 1
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_2.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
expect(subject.count_synced).to eq 2 expect(subject.count_synced).to eq 1
end
end end
context 'with selective sync' do context 'with selective sync by namespace' do
before do before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'delegates to #legacy_find_synced' do
expect(subject).to receive(:legacy_find_synced).and_call_original
subject.count_synced create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
end
it 'counts job artifacts that has been synced' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
end
expect(subject.count_synced).to eq 1 it 'counts job artifacts that has been synced' do
expect(subject.count_synced).to eq 2
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id) job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
expect(subject.count_synced).to eq 1 expect(subject.count_synced).to eq 1
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_1.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
expect(subject.count_synced).to eq 1 expect(subject.count_synced).to eq 1
end end
end end
end
describe '#count_failed' do context 'with selective sync by shard' do
it 'delegates to #legacy_find_failed' do before do
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(false) secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
expect(subject).to receive(:legacy_find_failed).and_call_original it 'counts job artifacts that has been synced' do
expect(subject.count_synced).to eq 2
end
subject.count_failed it 'ignores remote job artifacts' do
end job_artifact_5.update_column(:file_store, ObjectStorage::Store::REMOTE)
it 'delegates to #find_failed' do expect(subject.count_synced).to eq 1
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(true) end
expect(subject).to receive(:find_failed).and_call_original it 'ignores expired job artifacts' do
job_artifact_5.update_column(:expire_at, Date.yesterday)
subject.count_failed expect(subject.count_synced).to eq 1
end
end end
end
it 'counts job artifacts that sync has failed' do describe '#count_failed' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id) let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false) let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
end
expect(subject.count_failed).to eq 2 it 'counts job artifacts that sync has failed' do
end expect(subject.count_failed).to eq 3
end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, success: false) job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
expect(subject.count_failed).to eq 2 expect(subject.count_failed).to eq 2
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_1.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
expect(subject.count_failed).to eq 2 expect(subject.count_failed).to eq 2
end
end end
context 'with selective sync' do context 'with selective sync by namespace' do
before do before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'delegates to #legacy_find_failed' do create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
expect(subject).to receive(:legacy_find_failed).and_call_original create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
subject.count_failed create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
end end
it 'counts job artifacts that sync has failed' do it 'counts job artifacts that sync has failed' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) expect(subject.count_failed).to eq 2
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id) end
it 'ignores remote job artifacts' do
job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_failed).to eq 1 expect(subject.count_failed).to eq 1
end end
it 'does not count job artifacts of unsynced projects' do it 'ignores expired job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false) job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_failed).to eq 0 expect(subject.count_failed).to eq 1
end end
end
context 'with selective sync by shard' do
before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
it 'ignores remote job artifacts' do
job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, success: false)
end
it 'counts job artifacts that sync has failed' do
expect(subject.count_failed).to eq 2
end
it 'ignores remote job artifacts' do
job_artifact_5.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_failed).to eq 1 expect(subject.count_failed).to eq 1
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_5.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
expect(subject.count_failed).to eq 1 expect(subject.count_failed).to eq 1
end end
...@@ -241,179 +265,300 @@ describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -241,179 +265,300 @@ describe Geo::JobArtifactRegistryFinder, :geo do
end end
describe '#count_synced_missing_on_primary' do describe '#count_synced_missing_on_primary' do
it 'delegates to #legacy_find_synced_missing_on_primary' do let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(false) let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
expect(subject).to receive(:legacy_find_synced_missing_on_primary).and_call_original it 'counts job artifacts that have been synced and are missing on the primary' do
expect(subject.count_synced_missing_on_primary).to eq 2
end
subject.count_synced_missing_on_primary it 'ignores remote job artifacts' do
end job_artifact_3.update_column(:file_store, ObjectStorage::Store::REMOTE)
it 'delegates to #find_synced_missing_on_primary for PostgreSQL 10' do expect(subject.count_synced_missing_on_primary).to eq 1
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(true) end
expect(subject).to receive(:find_synced_missing_on_primary).and_call_original it 'ignores expired job artifacts' do
job_artifact_3.update_column(:expire_at, Date.yesterday)
subject.count_synced_missing_on_primary expect(subject.count_synced_missing_on_primary).to eq 1
end
end end
it 'counts job artifacts that have been synced and are missing on the primary' do context 'with selective sync by namespace' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true) before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject.count_synced_missing_on_primary).to eq 1 create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
end create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
it 'excludes job artifacts that are not missing on the primary' do it 'counts job artifacts that have been synced and are missing on the primary' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id) expect(subject.count_synced_missing_on_primary).to eq 2
end
expect(subject.count_synced_missing_on_primary).to eq 0 it 'ignores remote job artifacts' do
end job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
it 'excludes job artifacts that are not synced' do expect(subject.count_synced_missing_on_primary).to eq 1
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false, missing_on_primary: true) end
it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_synced_missing_on_primary).to eq 0 expect(subject.count_synced_missing_on_primary).to eq 1
end
end end
it 'ignores remote job artifacts' do context 'with selective sync by shard' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, missing_on_primary: true) before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, missing_on_primary: true)
end
expect(subject.count_synced_missing_on_primary).to eq 0 it 'counts job artifacts that have been synced and are missing on the primary' do
expect(subject.count_synced_missing_on_primary).to eq 2
end
it 'ignores remote job artifacts' do
job_artifact_5.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_synced_missing_on_primary).to eq 1
end
it 'ignores expired job artifacts' do
job_artifact_5.update_column(:expire_at, Date.yesterday)
expect(subject.count_synced_missing_on_primary).to eq 1
end
end end
end
it 'ignores expired job artifacts' do describe '#count_registry' do
job_artifact_1.update_column(:expire_at, Date.yesterday) let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true) let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
expect(subject.count_synced_missing_on_primary).to eq 0 it 'counts file registries for job artifacts' do
expect(subject.count_registry).to eq 4
end end
context 'with selective sync' do context 'with selective sync by namespace' do
before do before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end end
it 'delegates to #legacy_find_synced_missing_on_primary' do it 'counts file registries for job artifacts' do
expect(subject).to receive(:legacy_find_synced_missing_on_primary).and_call_original expect(subject.count_registry).to eq 2
end
end
context 'with selective sync by shard' do
before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'counts file registries for job artifacts' do
expect(subject.count_registry).to eq 1
end
end
end
end
subject.count_synced_missing_on_primary shared_examples 'finds all the things' do
describe '#find_unsynced' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
end end
it 'counts job artifacts that has been synced' do it 'returns job artifacts without an entry on the tracking database' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true) job_artifacts = subject.find_unsynced(batch_size: 10, except_artifact_ids: [job_artifact_2.id])
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
expect(subject.count_synced_missing_on_primary).to eq 2 expect(job_artifacts).to match_ids(job_artifact_4, job_artifact_5, job_artifact_6)
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, missing_on_primary: true) job_artifact_4.update_column(:file_store, ObjectStorage::Store::REMOTE)
job_artifacts = subject.find_unsynced(batch_size: 10)
expect(subject.count_synced_missing_on_primary).to eq 0 expect(job_artifacts).to match_ids(job_artifact_2, job_artifact_5, job_artifact_6)
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_5.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
job_artifacts = subject.find_unsynced(batch_size: 10)
expect(subject.count_synced_missing_on_primary).to eq 0 expect(job_artifacts).to match_ids(job_artifact_2, job_artifact_4, job_artifact_6)
end end
end end
end
end
shared_examples 'finds all the things' do context 'with selective sync by namespace' do
describe '#find_unsynced' do before do
it 'delegates to the correct method' do secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
expect(subject).to receive("#{method_prefix}_find_unsynced".to_sym).and_call_original end
subject.find_unsynced(batch_size: 10) it 'returns job artifacts without an entry on the tracking database' do
end create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true)
it 'returns job artifacts without an entry on the tracking database' do job_artifacts = subject.find_unsynced(batch_size: 10)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
job_artifacts = subject.find_unsynced(batch_size: 10) expect(job_artifacts).to match_ids(job_artifact_3)
end
expect(job_artifacts).to match_ids(job_artifact_2, job_artifact_4) it 'ignores remote job artifacts' do
end job_artifact_3.update_column(:file_store, ObjectStorage::Store::REMOTE)
it 'excludes job artifacts without an entry on the tracking database' do job_artifacts = subject.find_unsynced(batch_size: 10)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
job_artifacts = subject.find_unsynced(batch_size: 10, except_artifact_ids: [job_artifact_2.id]) expect(job_artifacts).to match_ids(job_artifact_1)
end
expect(job_artifacts).to match_ids(job_artifact_4) it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday)
job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_3)
end
end end
it 'ignores remote job artifacts' do context 'with selective sync by shard' do
job_artifact_2.update_column(:file_store, ObjectStorage::Store::REMOTE) before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
job_artifacts = subject.find_unsynced(batch_size: 10) it 'returns job artifacts without an entry on the tracking database' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: true)
expect(job_artifacts).to match_ids(job_artifact_4) job_artifacts = subject.find_unsynced(batch_size: 10)
end
it 'ignores expired job artifacts' do expect(job_artifacts).to match_ids(job_artifact_6)
job_artifact_2.update_column(:expire_at, Date.yesterday) end
it 'ignores remote job artifacts' do
job_artifact_6.update_column(:file_store, ObjectStorage::Store::REMOTE)
job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_5)
end
it 'ignores expired job artifacts' do
job_artifact_5.update_column(:expire_at, Date.yesterday)
job_artifacts = subject.find_unsynced(batch_size: 10) job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_4) expect(job_artifacts).to match_ids(job_artifact_6)
end
end end
end end
describe '#find_migrated_local' do describe '#find_migrated_local' do
it 'delegates to the correct method' do let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
expect(subject).to receive("#{method_prefix}_find_migrated_local".to_sym).and_call_original let!(:job_artifact_remote_1) { create(:ci_job_artifact, :remote_store, project: synced_project) }
let!(:job_artifact_remote_2) { create(:ci_job_artifact, :remote_store, project: unsynced_project) }
subject.find_migrated_local(batch_size: 10) let!(:job_artifact_remote_3) { create(:ci_job_artifact, :remote_store, project: project_broken_storage, expire_at: Date.yesterday) }
end
it 'returns job artifacts remotely and successfully synced locally' do it 'returns job artifacts remotely and successfully synced locally' do
job_artifact = create(:ci_job_artifact, :remote_store, project: synced_project) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10, except_artifact_ids: [job_artifact_remote_1.id])
expect(job_artifacts).to match_ids(job_artifact) expect(job_artifacts).to match_ids(job_artifact_remote_2)
end end
it 'excludes job artifacts stored remotely, but not synced yet' do it 'excludes synced job artifacts that are stored locally' do
create(:ci_job_artifact, :remote_store, project: synced_project) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to be_empty expect(job_artifacts).to match_ids(job_artifact_remote_1, job_artifact_remote_2)
end end
it 'excludes synced job artifacts that are stored locally' do it 'includes synced job artifacts that are expired' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_3.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to be_empty expect(job_artifacts).to match_ids(job_artifact_remote_3)
end end
it 'excludes except_artifact_ids' do context 'with selective sync by namespace' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id) before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
job_artifacts = subject.find_migrated_local(batch_size: 10, except_artifact_ids: [job_artifact_remote_1.id]) it 'returns job artifacts remotely and successfully synced locally' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
expect(job_artifacts).to match_ids(job_artifact_remote_2) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_remote_1)
end
end end
it 'includes synced job artifacts that are expired' do context 'with selective sync by shard' do
job_artifact = create(:ci_job_artifact, :remote_store, project: synced_project, expire_at: Date.yesterday) before do
create(:geo_job_artifact_registry, artifact_id: job_artifact.id) secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
job_artifacts = subject.find_migrated_local(batch_size: 10) it 'returns job artifacts remotely and successfully synced locally' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_3.id)
expect(job_artifacts).to match_ids(job_artifact) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_remote_3)
end
end end
end end
end end
......
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
View file @ cc088fff
...@@ -31,25 +31,6 @@ describe Geo::LfsObjectRegistryFinder, :geo do ...@@ -31,25 +31,6 @@ describe Geo::LfsObjectRegistryFinder, :geo do
stub_lfs_object_storage stub_lfs_object_storage
end end
it 'responds to file registry finder methods' do
file_registry_finder_methods = %i{
syncable
count_syncable
count_synced
count_failed
count_synced_missing_on_primary
count_registry
find_unsynced
find_migrated_local
find_retryable_failed_registries
find_retryable_synced_missing_on_primary_registries
}
file_registry_finder_methods.each do |method|
expect(subject).to respond_to(method)
end
end
shared_examples 'counts all the things' do shared_examples 'counts all the things' do
describe '#count_syncable' do describe '#count_syncable' do
before do before do
...@@ -661,34 +642,5 @@ describe Geo::LfsObjectRegistryFinder, :geo do ...@@ -661,34 +642,5 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end end
end end
# Disable transactions via :delete method because a foreign table it_behaves_like 'a file registry finder'
# can't see changes inside a transaction of a different connection.
context 'FDW', :geo_fdw, :delete do
context 'with use_fdw_queries_for_selective_sync disabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: false)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
context 'with use_fdw_queries_for_selective_sync enabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: true)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
end
context 'Legacy' do
before do
stub_fdw_disabled
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
end end
ee/spec/models/geo/fdw/ci/job_artifact_spec.rb 0 → 100644
View file @ cc088fff
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::Fdw::Ci::JobArtifact, :geo, type: :model do
context 'relationships' do
it { is_expected.to belong_to(:project).class_name('Geo::Fdw::Project').inverse_of(:job_artifacts) }
end
end
ee/spec/models/geo/fdw/project_spec.rb
View file @ cc088fff
...@@ -4,6 +4,7 @@ require 'spec_helper' ...@@ -4,6 +4,7 @@ require 'spec_helper'
RSpec.describe Geo::Fdw::Project, :geo, type: :model do RSpec.describe Geo::Fdw::Project, :geo, type: :model do
context 'relationships' do context 'relationships' do
it { is_expected.to have_many(:job_artifacts).class_name('Geo::Fdw::Ci::JobArtifact') }
it { is_expected.to have_many(:lfs_objects_projects).class_name('Geo::Fdw::LfsObjectsProject') } it { is_expected.to have_many(:lfs_objects_projects).class_name('Geo::Fdw::LfsObjectsProject') }
it { is_expected.to have_many(:lfs_objects).class_name('Geo::Fdw::LfsObject').through(:lfs_objects_projects) } it { is_expected.to have_many(:lfs_objects).class_name('Geo::Fdw::LfsObject').through(:lfs_objects_projects) }
end end
......
ee/spec/support/shared_examples/finders/file_registry_finders.rb
View file @ cc088fff
...@@ -18,23 +18,34 @@ shared_examples_for 'a file registry finder' do ...@@ -18,23 +18,34 @@ shared_examples_for 'a file registry finder' do
end end
end end
context 'FDW', :geo_fdw do # Disable transactions via :delete method because a foreign table
include_examples 'counts all the things' # can't see changes inside a transaction of a different connection.
context 'FDW', :geo_fdw, :delete do
context 'with use_fdw_queries_for_selective_sync disabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: false)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
include_examples 'finds all the things' do context 'with use_fdw_queries_for_selective_sync enabled' do
let(:method_prefix) { 'fdw' } before do
stub_feature_flags(use_fdw_queries_for_selective_sync: true)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end end
end end
context 'Legacy' do context 'Legacy' do
before do before do
allow(Gitlab::Geo::Fdw).to receive(:enabled?).and_return(false) stub_fdw_disabled
end end
include_examples 'counts all the things' include_examples 'counts all the things'
include_examples 'finds all the things'
include_examples 'finds all the things' do
let(:method_prefix) { 'legacy' }
end
end end
end end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7