Commit 23187fba authored by Douglas Barbosa Alexandre's avatar Douglas Barbosa Alexandre Committed by Mike Kozono

Remove method to find LFS objects migrated to object storage

The find_registry_differences method returns as unused
registries entries for LFS objects that are in object
storage when the object storage sync is disabled.

The Geo::Secondary::RegistryConsistencyWorker will
perform the cleanup of these unused registries.
parent 83f703c7
...@@ -44,7 +44,7 @@ module Geo ...@@ -44,7 +44,7 @@ module Geo
# #
# @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused # @return [Array] the first element is an Array of untracked IDs, and the second element is an Array of tracked IDs that are unused
def find_registry_differences(range) def find_registry_differences(range)
source_ids = lfs_objects..id_in(range).pluck_primary_key source_ids = lfs_objects.id_in(range).pluck_primary_key
tracked_ids = syncable.pluck_model_ids_in_range(range) tracked_ids = syncable.pluck_model_ids_in_range(range)
untracked_ids = source_ids - tracked_ids untracked_ids = source_ids - tracked_ids
...@@ -79,16 +79,6 @@ module Geo ...@@ -79,16 +79,6 @@ module Geo
alias_method :find_unsynced, :find_never_synced_registries alias_method :find_unsynced, :find_never_synced_registries
# rubocop:enable CodeReuse/ActiveRecord # rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord
def find_migrated_local(batch_size:, except_ids: [])
all_lfs_objects
.inner_join_registry
.with_files_stored_remotely
.id_not_in(except_ids)
.limit(batch_size)
end
# rubocop:enable CodeReuse/ActiveRecord
# rubocop:disable CodeReuse/ActiveRecord # rubocop:disable CodeReuse/ActiveRecord
def find_retryable_failed_registries(batch_size:, except_ids: []) def find_retryable_failed_registries(batch_size:, except_ids: [])
syncable syncable
......
...@@ -37,23 +37,12 @@ module Geo ...@@ -37,23 +37,12 @@ module Geo
end end
def find_migrated_local_objects(batch_size:) def find_migrated_local_objects(batch_size:)
lfs_object_ids = find_migrated_local_lfs_objects_ids(batch_size: batch_size)
attachment_ids = find_migrated_local_attachments_ids(batch_size: batch_size) attachment_ids = find_migrated_local_attachments_ids(batch_size: batch_size)
job_artifact_ids = find_migrated_local_job_artifacts_ids(batch_size: batch_size) job_artifact_ids = find_migrated_local_job_artifacts_ids(batch_size: batch_size)
take_batch(lfs_object_ids, attachment_ids, job_artifact_ids) take_batch(attachment_ids, job_artifact_ids)
end end
# rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local_lfs_objects_ids(batch_size:)
return [] unless lfs_objects_object_store_enabled?
lfs_objects_finder.find_migrated_local(batch_size: batch_size, except_ids: scheduled_file_ids(:lfs))
.pluck(Geo::Fdw::LfsObject.arel_table[:id])
.map { |id| ['lfs', id] }
end
# rubocop: enable CodeReuse/ActiveRecord
# rubocop: disable CodeReuse/ActiveRecord # rubocop: disable CodeReuse/ActiveRecord
def find_migrated_local_attachments_ids(batch_size:) def find_migrated_local_attachments_ids(batch_size:)
return [] unless attachments_object_store_enabled? return [] unless attachments_object_store_enabled?
...@@ -85,17 +74,12 @@ module Geo ...@@ -85,17 +74,12 @@ module Geo
FileUploader.object_store_enabled? FileUploader.object_store_enabled?
end end
def lfs_objects_object_store_enabled?
LfsObjectUploader.object_store_enabled?
end
def job_artifacts_object_store_enabled? def job_artifacts_object_store_enabled?
JobArtifactUploader.object_store_enabled? JobArtifactUploader.object_store_enabled?
end end
def object_store_enabled? def object_store_enabled?
attachments_object_store_enabled? || attachments_object_store_enabled? ||
lfs_objects_object_store_enabled? ||
job_artifacts_object_store_enabled? job_artifacts_object_store_enabled?
end end
...@@ -107,10 +91,6 @@ module Geo ...@@ -107,10 +91,6 @@ module Geo
@attachments_finder ||= AttachmentRegistryFinder.new(current_node_id: current_node.id) @attachments_finder ||= AttachmentRegistryFinder.new(current_node_id: current_node.id)
end end
def lfs_objects_finder
@lfs_objects_finder ||= LfsObjectRegistryFinder.new(current_node_id: current_node.id)
end
def job_artifacts_finder def job_artifacts_finder
@job_artifacts_finder ||= JobArtifactRegistryFinder.new(current_node_id: current_node.id) @job_artifacts_finder ||= JobArtifactRegistryFinder.new(current_node_id: current_node.id)
end end
......
...@@ -35,62 +35,6 @@ RSpec.describe Geo::MigratedLocalFilesCleanUpWorker, :geo, :geo_fdw, :use_sql_qu ...@@ -35,62 +35,6 @@ RSpec.describe Geo::MigratedLocalFilesCleanUpWorker, :geo, :geo_fdw, :use_sql_qu
subject.perform subject.perform
end end
context 'with LFS objects' do
let(:lfs_object_local) { create(:lfs_object) }
let(:lfs_object_remote_1) { create(:lfs_object, :object_storage) }
let(:lfs_object_remote_2) { create(:lfs_object, :object_storage) }
before do
stub_lfs_object_storage
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_local.id)
create(:geo_lfs_object_registry, lfs_object_id: lfs_object_remote_1.id)
end
it 'schedules worker for file stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('lfs', lfs_object_remote_1.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_local.id)
subject.perform
end
context 'with selective sync by namespace' do
let(:secondary) { create(:geo_node, :local_storage_only, selective_sync_type: 'namespaces', namespaces: [synced_group]) }
before do
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_local)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_1)
create(:lfs_objects_project, project: unsynced_project, lfs_object: lfs_object_remote_2)
end
it 'schedules worker for file stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('lfs', lfs_object_remote_1.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_remote_2.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_local.id)
subject.perform
end
end
context 'with selective sync by shard' do
let(:secondary) { create(:geo_node, :local_storage_only, selective_sync_type: 'shards', selective_sync_shards: ['broken']) }
before do
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_local)
create(:lfs_objects_project, project: project_broken_storage, lfs_object: lfs_object_remote_1)
create(:lfs_objects_project, project: synced_project, lfs_object: lfs_object_remote_2)
end
it 'schedules worker for file stored remotely and synced locally' do
expect(Geo::FileRegistryRemovalWorker).to receive(:perform_async).with('lfs', lfs_object_remote_1.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_remote_2.id)
expect(Geo::FileRegistryRemovalWorker).not_to receive(:perform_async).with(anything, lfs_object_local.id)
subject.perform
end
end
end
context 'with attachments' do context 'with attachments' do
let(:avatar_upload) { create(:upload) } let(:avatar_upload) { create(:upload) }
let(:personal_snippet_upload) { create(:upload, :personal_snippet_upload) } let(:personal_snippet_upload) { create(:upload, :personal_snippet_upload) }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment