Skip to content

G
gitlab-ce
Commit cc088fff authored by Nick Thomas's avatar Nick Thomas
Browse files
Options

Merge branch... 

Merge branch '11001-geo-implement-selective-sync-support-for-the-job-artifacts-fdw-queries' into 'master'

Geo - Add selective sync support for the job artifacts FDW queries

Closes #11001

See merge request gitlab-org/gitlab-ee!11892
parents e1c4d28e 79822cfa
Show whitespace changes
Inline Side-by-side
Showing with 573 additions and 466 deletions
ee/app/finders/geo/job_artifact_registry_finder.rb
View file @ cc088fff
...@@ -7,31 +7,29 @@ module Geo ...@@ -7,31 +7,29 @@ module Geo
end end
def count_synced def count_synced
if aggregate_pushdown_supported? job_artifacts_synced.count
find_synced.count
else
legacy_find_synced.count
end
end end
def count_failed def count_failed
if aggregate_pushdown_supported? job_artifacts_failed.count
find_failed.count
else
legacy_find_failed.count
end
end end
def count_synced_missing_on_primary def count_synced_missing_on_primary
if aggregate_pushdown_supported? job_artifacts_synced_missing_on_primary.count
find_synced_missing_on_primary.count
else
legacy_find_synced_missing_on_primary.count
end
end end
def count_registry def count_registry
Geo::JobArtifactRegistry.count registries_for_job_artifacts.count
end
def syncable
if use_legacy_queries_for_selective_sync?
legacy_finder.syncable
elsif selective_sync?
fdw_geo_node.job_artifacts.syncable
else
Ci::JobArtifact.syncable
end
end end
# Find limited amount of non replicated job artifacts. # Find limited amount of non replicated job artifacts.
...@@ -47,10 +45,10 @@ module Geo ...@@ -47,10 +45,10 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_unsynced(batch_size:, except_artifact_ids: []) def find_unsynced(batch_size:, except_artifact_ids: [])
relation = relation =
if use_legacy_queries? if use_legacy_queries_for_selective_sync?
legacy_find_unsynced(except_artifact_ids: except_artifact_ids) legacy_finder.job_artifacts_unsynced(except_artifact_ids: except_artifact_ids)
else else
fdw_find_unsynced(except_artifact_ids: except_artifact_ids) job_artifacts_unsynced(except_artifact_ids: except_artifact_ids)
end end
relation.limit(batch_size) relation.limit(batch_size)
...@@ -60,186 +58,98 @@ module Geo ...@@ -60,186 +58,98 @@ module Geo
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local(batch_size:, except_artifact_ids: []) def find_migrated_local(batch_size:, except_artifact_ids: [])
relation = relation =
if use_legacy_queries? if use_legacy_queries_for_selective_sync?
legacy_find_migrated_local(except_artifact_ids: except_artifact_ids) legacy_finder.job_artifacts_migrated_local(except_artifact_ids: except_artifact_ids)
else else
fdw_find_migrated_local(except_artifact_ids: except_artifact_ids) job_artifacts_migrated_local(except_artifact_ids: except_artifact_ids)
end end
relation.limit(batch_size) relation.limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
def syncable
all.geo_syncable
end
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_artifact_ids: []) def find_retryable_failed_registries(batch_size:, except_artifact_ids: [])
find_failed_registries Geo::JobArtifactRegistry
.failed
.retry_due .retry_due
.where.not(artifact_id: except_artifact_ids) .artifact_id_not_in(except_artifact_ids)
.limit(batch_size) .limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_retryable_synced_missing_on_primary_registries(batch_size:, except_artifact_ids: []) def find_retryable_synced_missing_on_primary_registries(batch_size:, except_artifact_ids: [])
find_synced_missing_on_primary_registries Geo::JobArtifactRegistry
.synced
.missing_on_primary
.retry_due .retry_due
.where.not(artifact_id: except_artifact_ids) .artifact_id_not_in(except_artifact_ids)
.limit(batch_size) .limit(batch_size)
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop: enable CodeReuse/ActiveRecord
private private
# rubocop: disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/Finder
def all def legacy_finder
if selective_sync? @legacy_finder ||= Geo::LegacyJobArtifactRegistryFinder.new(current_node: current_node)
Ci::JobArtifact.joins(:project).where(projects: { id: current_node.projects })
else
Ci::JobArtifact.all
end
end end
# rubocop: enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/Finder
def find_synced def fdw_geo_node
if use_legacy_queries? @fdw_geo_node ||= Geo::Fdw::GeoNode.find(current_node.id)
legacy_find_synced
else
fdw_find.merge(find_synced_registries)
end
end end
def find_synced_missing_on_primary def registries_for_job_artifacts
if use_legacy_queries? if use_legacy_queries_for_selective_sync?
legacy_find_synced_missing_on_primary legacy_finder.registries_for_job_artifacts
else else
fdw_find.merge(find_synced_missing_on_primary_registries) fdw_geo_node
.job_artifacts
.inner_join_job_artifact_registry
.syncable
end end
end end
def find_failed def job_artifacts_synced
if use_legacy_queries? if use_legacy_queries_for_selective_sync?
legacy_find_failed legacy_finder.job_artifacts_synced
else else
fdw_find.merge(find_failed_registries) registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.synced)
end
end
def find_synced_registries
Geo::JobArtifactRegistry.synced
end
def find_synced_missing_on_primary_registries
find_synced_registries.missing_on_primary
end
def find_failed_registries
Geo::JobArtifactRegistry.failed
end
#
# FDW accessors
#
# rubocop: disable CodeReuse/ActiveRecord
def fdw_find
fdw_all.joins("INNER JOIN job_artifact_registry ON job_artifact_registry.artifact_id = #{fdw_table}.id")
.geo_syncable
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def fdw_find_unsynced(except_artifact_ids:)
fdw_all.joins("LEFT OUTER JOIN job_artifact_registry
ON job_artifact_registry.artifact_id = #{fdw_table}.id")
.geo_syncable
.where(job_artifact_registry: { artifact_id: nil })
.where.not(id: except_artifact_ids)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def fdw_find_migrated_local(except_artifact_ids:)
fdw_all.joins("INNER JOIN job_artifact_registry ON job_artifact_registry.artifact_id = #{fdw_table}.id")
.with_files_stored_remotely
.where.not(id: except_artifact_ids)
.merge(Geo::JobArtifactRegistry.all)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord def job_artifacts_failed
def fdw_all if use_legacy_queries_for_selective_sync?
if selective_sync? legacy_finder.job_artifacts_failed
Geo::Fdw::Ci::JobArtifact.joins(:project).where(projects: { id: current_node.projects })
else else
Geo::Fdw::Ci::JobArtifact.all registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.failed)
end
end end
# rubocop: enable CodeReuse/ActiveRecord
def fdw_table
Geo::Fdw::Ci::JobArtifact.table_name
end end
# def job_artifacts_synced_missing_on_primary
# Legacy accessors (non FDW) if use_legacy_queries_for_selective_sync?
# legacy_finder.job_artifacts_synced_missing_on_primary
else
# rubocop: disable CodeReuse/ActiveRecord registries_for_job_artifacts.merge(Geo::JobArtifactRegistry.synced.missing_on_primary)
def legacy_find_synced
legacy_inner_join_registry_ids(
syncable,
find_synced_registries.pluck(:artifact_id),
Ci::JobArtifact
)
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def legacy_find_failed
legacy_inner_join_registry_ids(
syncable,
find_failed_registries.pluck(:artifact_id),
Ci::JobArtifact
)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def legacy_find_unsynced(except_artifact_ids:)
registry_artifact_ids = Geo::JobArtifactRegistry.pluck(:artifact_id) | except_artifact_ids
legacy_left_outer_join_registry_ids(
syncable,
registry_artifact_ids,
Ci::JobArtifact
)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord
def legacy_find_migrated_local(except_artifact_ids:)
registry_artifact_ids = Geo::JobArtifactRegistry.pluck(:artifact_id) - except_artifact_ids
legacy_inner_join_registry_ids( def job_artifacts_unsynced(except_artifact_ids:)
all.with_files_stored_remotely, fdw_geo_node
registry_artifact_ids, .job_artifacts
Ci::JobArtifact .syncable
) .missing_job_artifact_registry
.id_not_in(except_artifact_ids)
end end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord def job_artifacts_migrated_local(except_artifact_ids:)
def legacy_find_synced_missing_on_primary fdw_geo_node
legacy_inner_join_registry_ids( .job_artifacts
syncable, .inner_join_job_artifact_registry
find_synced_missing_on_primary_registries.pluck(:artifact_id), .with_files_stored_remotely
Ci::JobArtifact .id_not_in(except_artifact_ids)
)
end end
# rubocop: enable CodeReuse/ActiveRecord
end end
end end
ee/app/finders/geo/legacy_job_artifact_registry_finder.rb 0 → 100644
View file @ cc088fff
# frozen_string_literal: true
module Geo
class LegacyJobArtifactRegistryFinder < RegistryFinder
def syncable
current_node.job_artifacts.syncable
end
def job_artifacts_synced
legacy_inner_join_registry_ids(
syncable,
Geo::JobArtifactRegistry.synced.pluck_artifact_key,
Ci::JobArtifact
)
end
def job_artifacts_failed
legacy_inner_join_registry_ids(
syncable,
Geo::JobArtifactRegistry.failed.pluck_artifact_key,
Ci::JobArtifact
)
end
def job_artifacts_synced_missing_on_primary
legacy_inner_join_registry_ids(
syncable,
Geo::JobArtifactRegistry.synced.missing_on_primary.pluck_artifact_key,
Ci::JobArtifact
)
end
def job_artifacts_unsynced(except_artifact_ids: [])
registry_artifact_ids = Geo::JobArtifactRegistry.pluck_artifact_key | except_artifact_ids
legacy_left_outer_join_registry_ids(
syncable,
registry_artifact_ids,
Ci::JobArtifact
)
end
def job_artifacts_migrated_local(except_artifact_ids: [])
registry_artifact_ids = Geo::JobArtifactRegistry.pluck_artifact_key - except_artifact_ids
legacy_inner_join_registry_ids(
current_node.job_artifacts.with_files_stored_remotely,
registry_artifact_ids,
Ci::JobArtifact
)
end
def registries_for_job_artifacts
return Geo::JobArtifactRegistry.all unless selective_sync?
legacy_inner_join_registry_ids(
Geo::JobArtifactRegistry.all,
current_node.job_artifacts.pluck_primary_key,
Geo::JobArtifactRegistry,
foreign_key: :artifact_id
)
end
end
end
ee/app/models/ee/ci/job_artifact.rb
View file @ cc088fff
...@@ -16,7 +16,8 @@ module EE ...@@ -16,7 +16,8 @@ module EE
METRICS_REPORT_FILE_TYPES = %w[metrics].freeze METRICS_REPORT_FILE_TYPES = %w[metrics].freeze
scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
scope :geo_syncable, -> { with_files_stored_locally.not_expired } scope :project_id_in, ->(ids) { joins(:project).merge(::Project.id_in(ids)) }
scope :syncable, -> { with_files_stored_locally.not_expired }
scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) } scope :with_files_stored_remotely, -> { where(file_store: ::JobArtifactUploader::Store::REMOTE) }
scope :security_reports, -> do scope :security_reports, -> do
......
ee/app/models/geo/fdw/ci/job_artifact.rb
View file @ cc088fff
...@@ -10,8 +10,42 @@ module Geo ...@@ -10,8 +10,42 @@ module Geo
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('ci_job_artifacts') self.table_name = Gitlab::Geo::Fdw.foreign_table_name('ci_job_artifacts')
belongs_to :project, class_name: 'Geo::Fdw::Project', inverse_of: :job_artifacts
scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) } scope :not_expired, -> { where('expire_at IS NULL OR expire_at > ?', Time.current) }
scope :geo_syncable, -> { with_files_stored_locally.not_expired } scope :project_id_in, ->(ids) { joins(:project).merge(Geo::Fdw::Project.id_in(ids)) }
scope :syncable, -> { with_files_stored_locally.not_expired }
class << self
def inner_join_job_artifact_registry
join_statement =
arel_table
.join(job_artifact_registry_table, Arel::Nodes::InnerJoin)
.on(arel_table[:id].eq(job_artifact_registry_table[:artifact_id]))
joins(join_statement.join_sources)
end
def missing_job_artifact_registry
left_outer_join_job_artifact_registry
.where(job_artifact_registry_table[:id].eq(nil))
end
private
def job_artifact_registry_table
Geo::JobArtifactRegistry.arel_table
end
def left_outer_join_job_artifact_registry
join_statement =
arel_table
.join(job_artifact_registry_table, Arel::Nodes::OuterJoin)
.on(arel_table[:id].eq(job_artifact_registry_table[:artifact_id]))
joins(join_statement.join_sources)
end
end
end end
end end
end end
......
ee/app/models/geo/fdw/geo_node.rb
View file @ cc088fff
...@@ -14,6 +14,12 @@ module Geo ...@@ -14,6 +14,12 @@ module Geo
has_many :geo_node_namespace_links, class_name: 'Geo::Fdw::GeoNodeNamespaceLink' has_many :geo_node_namespace_links, class_name: 'Geo::Fdw::GeoNodeNamespaceLink'
has_many :namespaces, class_name: 'Geo::Fdw::Namespace', through: :geo_node_namespace_links has_many :namespaces, class_name: 'Geo::Fdw::Namespace', through: :geo_node_namespace_links
def job_artifacts
Geo::Fdw::Ci::JobArtifact.all unless selective_sync?
Geo::Fdw::Ci::JobArtifact.project_id_in(projects)
end
def lfs_objects def lfs_objects
return Geo::Fdw::LfsObject.all unless selective_sync? return Geo::Fdw::LfsObject.all unless selective_sync?
......
ee/app/models/geo/fdw/project.rb
View file @ cc088fff
...@@ -7,6 +7,7 @@ module Geo ...@@ -7,6 +7,7 @@ module Geo
self.table_name = Gitlab::Geo::Fdw.foreign_table_name('projects') self.table_name = Gitlab::Geo::Fdw.foreign_table_name('projects')
has_many :job_artifacts, class_name: 'Geo::Fdw::Ci::JobArtifact'
has_many :lfs_objects_projects, class_name: 'Geo::Fdw::LfsObjectsProject' has_many :lfs_objects_projects, class_name: 'Geo::Fdw::LfsObjectsProject'
has_many :lfs_objects, class_name: 'Geo::Fdw::LfsObject', through: :lfs_objects_projects has_many :lfs_objects, class_name: 'Geo::Fdw::LfsObject', through: :lfs_objects_projects
......
ee/app/models/geo/job_artifact_registry.rb
View file @ cc088fff
...@@ -2,4 +2,12 @@ ...@@ -2,4 +2,12 @@
class Geo::JobArtifactRegistry < Geo::BaseRegistry class Geo::JobArtifactRegistry < Geo::BaseRegistry
include Geo::Syncable include Geo::Syncable
def self.artifact_id_not_in(ids)
where.not(artifact_id: ids)
end
def self.pluck_artifact_key
where(nil).pluck(:artifact_id)
end
end end
ee/app/models/geo_node.rb
View file @ cc088fff
...@@ -198,6 +198,12 @@ class GeoNode < ApplicationRecord ...@@ -198,6 +198,12 @@ class GeoNode < ApplicationRecord
end end
end end
def job_artifacts
Ci::JobArtifact.all unless selective_sync?
Ci::JobArtifact.project_id_in(projects)
end
def lfs_objects def lfs_objects
return LfsObject.all unless selective_sync? return LfsObject.all unless selective_sync?
......
ee/changelogs/unreleased/11001-geo-implement-selective-sync-support-for-the-job-artifacts-fdw-queries.yml 0 → 100644
View file @ cc088fff
---
title: Geo - Add selective sync support for the job artifacts FDW queries
merge_request: 11892
author:
type: changed
ee/spec/finders/geo/attachment_registry_finder_spec.rb
View file @ cc088fff
...@@ -28,25 +28,6 @@ describe Geo::AttachmentRegistryFinder, :geo do ...@@ -28,25 +28,6 @@ describe Geo::AttachmentRegistryFinder, :geo do
stub_current_geo_node(secondary) stub_current_geo_node(secondary)
end end
it 'responds to file registry finder methods' do
file_registry_finder_methods = %i{
syncable
count_syncable
count_synced
count_failed
count_synced_missing_on_primary
count_registry
find_unsynced
find_migrated_local
find_retryable_failed_registries
find_retryable_synced_missing_on_primary_registries
}
file_registry_finder_methods.each do |method|
expect(subject).to respond_to(method)
end
end
shared_examples 'finds all the things' do shared_examples 'finds all the things' do
describe '#find_unsynced' do describe '#find_unsynced' do
let!(:upload_1) { create(:upload, model: synced_group) } let!(:upload_1) { create(:upload, model: synced_group) }
...@@ -475,32 +456,5 @@ describe Geo::AttachmentRegistryFinder, :geo do ...@@ -475,32 +456,5 @@ describe Geo::AttachmentRegistryFinder, :geo do
end end
end end
context 'FDW', :geo_fdw do it_behaves_like 'a file registry finder'
context 'with use_fdw_queries_for_selective_sync disabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: false)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
context 'with use_fdw_queries_for_selective_sync enabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: true)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
end
context 'Legacy' do
before do
stub_fdw_disabled
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
end end
ee/spec/finders/geo/job_artifact_registry_finder_spec.rb
View file @ cc088fff
...@@ -11,13 +11,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -11,13 +11,7 @@ describe Geo::JobArtifactRegistryFinder, :geo do
let(:synced_group) { create(:group) } let(:synced_group) { create(:group) }
let(:synced_project) { create(:project, group: synced_group) } let(:synced_project) { create(:project, group: synced_group) }
let(:unsynced_project) { create(:project) } let(:unsynced_project) { create(:project) }
let(:project_broken_storage) { create(:project, :broken_storage) }
let(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let(:job_artifact_remote_1) { create(:ci_job_artifact, :remote_store, project: synced_project) }
let(:job_artifact_remote_2) { create(:ci_job_artifact, :remote_store, project: unsynced_project) }
subject { described_class.new(current_node: secondary) } subject { described_class.new(current_node: secondary) }
...@@ -28,30 +22,30 @@ describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -28,30 +22,30 @@ describe Geo::JobArtifactRegistryFinder, :geo do
shared_examples 'counts all the things' do shared_examples 'counts all the things' do
describe '#count_syncable' do describe '#count_syncable' do
before do let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
job_artifact_1 let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
job_artifact_2 let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
job_artifact_3 let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
job_artifact_4 let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
end let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
it 'counts job artifacts' do it 'counts job artifacts' do
expect(subject.count_syncable).to eq 4 expect(subject.count_syncable).to eq 6
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE) job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_syncable).to eq 3 expect(subject.count_syncable).to eq 5
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_syncable).to eq 3 expect(subject.count_syncable).to eq 5
end end
context 'with selective sync' do context 'with selective sync by namespace' do
before do before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end end
...@@ -72,168 +66,198 @@ describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -72,168 +66,198 @@ describe Geo::JobArtifactRegistryFinder, :geo do
expect(subject.count_syncable).to eq 1 expect(subject.count_syncable).to eq 1
end end
end end
context 'with selective sync by shard' do
before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end end
describe '#count_synced' do it 'counts job artifacts' do
it 'delegates to #legacy_find_synced' do expect(subject.count_syncable).to eq 2
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(false) end
expect(subject).to receive(:legacy_find_synced).and_call_original it 'ignores remote job artifacts' do
job_artifact_5.update_column(:file_store, ObjectStorage::Store::REMOTE)
subject.count_synced expect(subject.count_syncable).to eq 1
end end
it 'delegates to #find_synced for PostgreSQL 10' do it 'ignores expired job artifacts' do
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(true) job_artifact_5.update_column(:expire_at, Date.yesterday)
expect(subject).to receive(:find_synced).and_call_original
subject.count_synced expect(subject.count_syncable).to eq 1
end
end
end end
it 'counts job artifacts that have been synced' do describe '#count_synced' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
end
it 'counts job artifacts that have been synced' do
expect(subject.count_synced).to eq 2 expect(subject.count_synced).to eq 2
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id) job_artifact_2.update_column(:file_store, ObjectStorage::Store::REMOTE)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
expect(subject.count_synced).to eq 2 expect(subject.count_synced).to eq 1
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_2.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
expect(subject.count_synced).to eq 2 expect(subject.count_synced).to eq 1
end
end end
context 'with selective sync' do context 'with selective sync by namespace' do
before do before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'delegates to #legacy_find_synced' do create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
expect(subject).to receive(:legacy_find_synced).and_call_original
subject.count_synced
end
it 'counts job artifacts that has been synced' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
end
expect(subject.count_synced).to eq 1 it 'counts job artifacts that has been synced' do
expect(subject.count_synced).to eq 2
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id) job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
expect(subject.count_synced).to eq 1 expect(subject.count_synced).to eq 1
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_1.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id)
expect(subject.count_synced).to eq 1 expect(subject.count_synced).to eq 1
end end
end end
end
describe '#count_failed' do context 'with selective sync by shard' do
it 'delegates to #legacy_find_failed' do before do
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(false) secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
expect(subject).to receive(:legacy_find_failed).and_call_original create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
subject.count_failed it 'counts job artifacts that has been synced' do
expect(subject.count_synced).to eq 2
end end
it 'delegates to #find_failed' do it 'ignores remote job artifacts' do
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(true) job_artifact_5.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject).to receive(:find_failed).and_call_original expect(subject.count_synced).to eq 1
end
subject.count_failed it 'ignores expired job artifacts' do
job_artifact_5.update_column(:expire_at, Date.yesterday)
expect(subject.count_synced).to eq 1
end
end
end end
it 'counts job artifacts that sync has failed' do describe '#count_failed' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
end
expect(subject.count_failed).to eq 2 it 'counts job artifacts that sync has failed' do
expect(subject.count_failed).to eq 3
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, success: false) job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
expect(subject.count_failed).to eq 2 expect(subject.count_failed).to eq 2
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_1.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
expect(subject.count_failed).to eq 2 expect(subject.count_failed).to eq 2
end end
end
context 'with selective sync' do context 'with selective sync by namespace' do
before do before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'delegates to #legacy_find_failed' do create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
expect(subject).to receive(:legacy_find_failed).and_call_original create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
subject.count_failed create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
end end
it 'counts job artifacts that sync has failed' do it 'counts job artifacts that sync has failed' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) expect(subject.count_failed).to eq 2
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id) end
it 'ignores remote job artifacts' do
job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_failed).to eq 1 expect(subject.count_failed).to eq 1
end end
it 'does not count job artifacts of unsynced projects' do it 'ignores expired job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false) job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_failed).to eq 0 expect(subject.count_failed).to eq 1
end
end end
it 'ignores remote job artifacts' do context 'with selective sync by shard' do
job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE) before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false) create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, success: false)
end
it 'counts job artifacts that sync has failed' do
expect(subject.count_failed).to eq 2
end
it 'ignores remote job artifacts' do
job_artifact_5.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_failed).to eq 1 expect(subject.count_failed).to eq 1
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_5.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
expect(subject.count_failed).to eq 1 expect(subject.count_failed).to eq 1
end end
...@@ -241,179 +265,300 @@ describe Geo::JobArtifactRegistryFinder, :geo do ...@@ -241,179 +265,300 @@ describe Geo::JobArtifactRegistryFinder, :geo do
end end
describe '#count_synced_missing_on_primary' do describe '#count_synced_missing_on_primary' do
it 'delegates to #legacy_find_synced_missing_on_primary' do let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(false) let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
expect(subject).to receive(:legacy_find_synced_missing_on_primary).and_call_original it 'counts job artifacts that have been synced and are missing on the primary' do
expect(subject.count_synced_missing_on_primary).to eq 2
end
subject.count_synced_missing_on_primary it 'ignores remote job artifacts' do
job_artifact_3.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_synced_missing_on_primary).to eq 1
end end
it 'delegates to #find_synced_missing_on_primary for PostgreSQL 10' do it 'ignores expired job artifacts' do
allow(subject).to receive(:aggregate_pushdown_supported?).and_return(true) job_artifact_3.update_column(:expire_at, Date.yesterday)
expect(subject.count_synced_missing_on_primary).to eq 1
end
end
expect(subject).to receive(:find_synced_missing_on_primary).and_call_original context 'with selective sync by namespace' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
subject.count_synced_missing_on_primary create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end end
it 'counts job artifacts that have been synced and are missing on the primary' do it 'counts job artifacts that have been synced and are missing on the primary' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true) expect(subject.count_synced_missing_on_primary).to eq 2
end
it 'ignores remote job artifacts' do
job_artifact_1.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_synced_missing_on_primary).to eq 1 expect(subject.count_synced_missing_on_primary).to eq 1
end end
it 'excludes job artifacts that are not missing on the primary' do it 'ignores expired job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id) job_artifact_1.update_column(:expire_at, Date.yesterday)
expect(subject.count_synced_missing_on_primary).to eq 0 expect(subject.count_synced_missing_on_primary).to eq 1
end end
end
context 'with selective sync by shard' do
before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
it 'excludes job artifacts that are not synced' do create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false, missing_on_primary: true) create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id, missing_on_primary: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id, missing_on_primary: true)
end
expect(subject.count_synced_missing_on_primary).to eq 0 it 'counts job artifacts that have been synced and are missing on the primary' do
expect(subject.count_synced_missing_on_primary).to eq 2
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, missing_on_primary: true) job_artifact_5.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_synced_missing_on_primary).to eq 0 expect(subject.count_synced_missing_on_primary).to eq 1
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_5.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
expect(subject.count_synced_missing_on_primary).to eq 1
end
end
end
describe '#count_registry' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: false)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_4.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_6.id)
end
expect(subject.count_synced_missing_on_primary).to eq 0 it 'counts file registries for job artifacts' do
expect(subject.count_registry).to eq 4
end end
context 'with selective sync' do context 'with selective sync by namespace' do
before do before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group]) secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end end
it 'delegates to #legacy_find_synced_missing_on_primary' do it 'counts file registries for job artifacts' do
expect(subject).to receive(:legacy_find_synced_missing_on_primary).and_call_original expect(subject.count_registry).to eq 2
end
end
subject.count_synced_missing_on_primary context 'with selective sync by shard' do
before do
secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end end
it 'counts job artifacts that has been synced' do it 'counts file registries for job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true) expect(subject.count_registry).to eq 1
create(:geo_job_artifact_registry, artifact_id: job_artifact_2.id, missing_on_primary: true) end
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, missing_on_primary: true) end
end
end
expect(subject.count_synced_missing_on_primary).to eq 2 shared_examples 'finds all the things' do
describe '#find_unsynced' do
let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_2) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_3) { create(:ci_job_artifact, project: synced_project) }
let!(:job_artifact_4) { create(:ci_job_artifact, project: unsynced_project) }
let!(:job_artifact_5) { create(:ci_job_artifact, project: project_broken_storage) }
let!(:job_artifact_6) { create(:ci_job_artifact, project: project_broken_storage) }
context 'without selective sync' do
before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
end
it 'returns job artifacts without an entry on the tracking database' do
job_artifacts = subject.find_unsynced(batch_size: 10, except_artifact_ids: [job_artifact_2.id])
expect(job_artifacts).to match_ids(job_artifact_4, job_artifact_5, job_artifact_6)
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id, missing_on_primary: true) job_artifact_4.update_column(:file_store, ObjectStorage::Store::REMOTE)
expect(subject.count_synced_missing_on_primary).to eq 0 job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_2, job_artifact_5, job_artifact_6)
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_1.update_column(:expire_at, Date.yesterday) job_artifact_5.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, missing_on_primary: true)
expect(subject.count_synced_missing_on_primary).to eq 0 job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_2, job_artifact_4, job_artifact_6)
end end
end end
context 'with selective sync by namespace' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end end
it 'returns job artifacts without an entry on the tracking database' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true)
job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_3)
end end
shared_examples 'finds all the things' do it 'ignores remote job artifacts' do
describe '#find_unsynced' do job_artifact_3.update_column(:file_store, ObjectStorage::Store::REMOTE)
it 'delegates to the correct method' do
expect(subject).to receive("#{method_prefix}_find_unsynced".to_sym).and_call_original job_artifacts = subject.find_unsynced(batch_size: 10)
subject.find_unsynced(batch_size: 10) expect(job_artifacts).to match_ids(job_artifact_1)
end end
it 'returns job artifacts without an entry on the tracking database' do it 'ignores expired job artifacts' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true) job_artifact_1.update_column(:expire_at, Date.yesterday)
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false)
job_artifacts = subject.find_unsynced(batch_size: 10) job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_2, job_artifact_4) expect(job_artifacts).to match_ids(job_artifact_3)
end
end end
it 'excludes job artifacts without an entry on the tracking database' do context 'with selective sync by shard' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id, success: true) before do
create(:geo_job_artifact_registry, artifact_id: job_artifact_3.id, success: false) secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
job_artifacts = subject.find_unsynced(batch_size: 10, except_artifact_ids: [job_artifact_2.id]) it 'returns job artifacts without an entry on the tracking database' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_5.id, success: true)
expect(job_artifacts).to match_ids(job_artifact_4) job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_6)
end end
it 'ignores remote job artifacts' do it 'ignores remote job artifacts' do
job_artifact_2.update_column(:file_store, ObjectStorage::Store::REMOTE) job_artifact_6.update_column(:file_store, ObjectStorage::Store::REMOTE)
job_artifacts = subject.find_unsynced(batch_size: 10) job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_4) expect(job_artifacts).to match_ids(job_artifact_5)
end end
it 'ignores expired job artifacts' do it 'ignores expired job artifacts' do
job_artifact_2.update_column(:expire_at, Date.yesterday) job_artifact_5.update_column(:expire_at, Date.yesterday)
job_artifacts = subject.find_unsynced(batch_size: 10) job_artifacts = subject.find_unsynced(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_4) expect(job_artifacts).to match_ids(job_artifact_6)
end
end end
end end
describe '#find_migrated_local' do describe '#find_migrated_local' do
it 'delegates to the correct method' do let!(:job_artifact_1) { create(:ci_job_artifact, project: synced_project) }
expect(subject).to receive("#{method_prefix}_find_migrated_local".to_sym).and_call_original let!(:job_artifact_remote_1) { create(:ci_job_artifact, :remote_store, project: synced_project) }
let!(:job_artifact_remote_2) { create(:ci_job_artifact, :remote_store, project: unsynced_project) }
subject.find_migrated_local(batch_size: 10) let!(:job_artifact_remote_3) { create(:ci_job_artifact, :remote_store, project: project_broken_storage, expire_at: Date.yesterday) }
end
it 'returns job artifacts remotely and successfully synced locally' do it 'returns job artifacts remotely and successfully synced locally' do
job_artifact = create(:ci_job_artifact, :remote_store, project: synced_project) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10, except_artifact_ids: [job_artifact_remote_1.id])
expect(job_artifacts).to match_ids(job_artifact) expect(job_artifacts).to match_ids(job_artifact_remote_2)
end end
it 'excludes job artifacts stored remotely, but not synced yet' do it 'excludes synced job artifacts that are stored locally' do
create(:ci_job_artifact, :remote_store, project: synced_project) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to be_empty expect(job_artifacts).to match_ids(job_artifact_remote_1, job_artifact_remote_2)
end end
it 'excludes synced job artifacts that are stored locally' do it 'includes synced job artifacts that are expired' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_1.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_3.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to be_empty expect(job_artifacts).to match_ids(job_artifact_remote_3)
end end
it 'excludes except_artifact_ids' do context 'with selective sync by namespace' do
before do
secondary.update!(selective_sync_type: 'namespaces', namespaces: [synced_group])
end
it 'returns job artifacts remotely and successfully synced locally' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id) create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_2.id)
job_artifacts = subject.find_migrated_local(batch_size: 10, except_artifact_ids: [job_artifact_remote_1.id]) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact_remote_2) expect(job_artifacts).to match_ids(job_artifact_remote_1)
end
end end
it 'includes synced job artifacts that are expired' do context 'with selective sync by shard' do
job_artifact = create(:ci_job_artifact, :remote_store, project: synced_project, expire_at: Date.yesterday) before do
create(:geo_job_artifact_registry, artifact_id: job_artifact.id) secondary.update!(selective_sync_type: 'shards', selective_sync_shards: ['broken'])
end
it 'returns job artifacts remotely and successfully synced locally' do
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_1.id)
create(:geo_job_artifact_registry, artifact_id: job_artifact_remote_3.id)
job_artifacts = subject.find_migrated_local(batch_size: 10) job_artifacts = subject.find_migrated_local(batch_size: 10)
expect(job_artifacts).to match_ids(job_artifact) expect(job_artifacts).to match_ids(job_artifact_remote_3)
end
end end
end end
end end
......
ee/spec/finders/geo/lfs_object_registry_finder_spec.rb
View file @ cc088fff
...@@ -31,25 +31,6 @@ describe Geo::LfsObjectRegistryFinder, :geo do ...@@ -31,25 +31,6 @@ describe Geo::LfsObjectRegistryFinder, :geo do
stub_lfs_object_storage stub_lfs_object_storage
end end
it 'responds to file registry finder methods' do
file_registry_finder_methods = %i{
syncable
count_syncable
count_synced
count_failed
count_synced_missing_on_primary
count_registry
find_unsynced
find_migrated_local
find_retryable_failed_registries
find_retryable_synced_missing_on_primary_registries
}
file_registry_finder_methods.each do |method|
expect(subject).to respond_to(method)
end
end
shared_examples 'counts all the things' do shared_examples 'counts all the things' do
describe '#count_syncable' do describe '#count_syncable' do
before do before do
...@@ -661,34 +642,5 @@ describe Geo::LfsObjectRegistryFinder, :geo do ...@@ -661,34 +642,5 @@ describe Geo::LfsObjectRegistryFinder, :geo do
end end
end end
# Disable transactions via :delete method because a foreign table it_behaves_like 'a file registry finder'
# can't see changes inside a transaction of a different connection.
context 'FDW', :geo_fdw, :delete do
context 'with use_fdw_queries_for_selective_sync disabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: false)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
context 'with use_fdw_queries_for_selective_sync enabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: true)
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
end
context 'Legacy' do
before do
stub_fdw_disabled
end
include_examples 'counts all the things'
include_examples 'finds all the things'
end
end end
ee/spec/models/geo/fdw/ci/job_artifact_spec.rb 0 → 100644
View file @ cc088fff
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Geo::Fdw::Ci::JobArtifact, :geo, type: :model do
context 'relationships' do
it { is_expected.to belong_to(:project).class_name('Geo::Fdw::Project').inverse_of(:job_artifacts) }
end
end
ee/spec/models/geo/fdw/project_spec.rb
View file @ cc088fff
...@@ -4,6 +4,7 @@ require 'spec_helper' ...@@ -4,6 +4,7 @@ require 'spec_helper'
RSpec.describe Geo::Fdw::Project, :geo, type: :model do RSpec.describe Geo::Fdw::Project, :geo, type: :model do
context 'relationships' do context 'relationships' do
it { is_expected.to have_many(:job_artifacts).class_name('Geo::Fdw::Ci::JobArtifact') }
it { is_expected.to have_many(:lfs_objects_projects).class_name('Geo::Fdw::LfsObjectsProject') } it { is_expected.to have_many(:lfs_objects_projects).class_name('Geo::Fdw::LfsObjectsProject') }
it { is_expected.to have_many(:lfs_objects).class_name('Geo::Fdw::LfsObject').through(:lfs_objects_projects) } it { is_expected.to have_many(:lfs_objects).class_name('Geo::Fdw::LfsObject').through(:lfs_objects_projects) }
end end
......
ee/spec/support/shared_examples/finders/file_registry_finders.rb
View file @ cc088fff
...@@ -18,23 +18,34 @@ shared_examples_for 'a file registry finder' do ...@@ -18,23 +18,34 @@ shared_examples_for 'a file registry finder' do
end end
end end
context 'FDW', :geo_fdw do # Disable transactions via :delete method because a foreign table
# can't see changes inside a transaction of a different connection.
context 'FDW', :geo_fdw, :delete do
context 'with use_fdw_queries_for_selective_sync disabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: false)
end
include_examples 'counts all the things' include_examples 'counts all the things'
include_examples 'finds all the things'
end
context 'with use_fdw_queries_for_selective_sync enabled' do
before do
stub_feature_flags(use_fdw_queries_for_selective_sync: true)
end
include_examples 'finds all the things' do include_examples 'counts all the things'
let(:method_prefix) { 'fdw' } include_examples 'finds all the things'
end end
end end
context 'Legacy' do context 'Legacy' do
before do before do
allow(Gitlab::Geo::Fdw).to receive(:enabled?).and_return(false) stub_fdw_disabled
end end
include_examples 'counts all the things' include_examples 'counts all the things'
include_examples 'finds all the things'
include_examples 'finds all the things' do
let(:method_prefix) { 'legacy' }
end
end end
end end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7