Skip already downloaded LFS objects in mirror updates

Previously a project mirror with linked LFS objects would re-download
files again. This would lead to unnecessary HTTPS traffic between the
upstream mirror and GitLab.

We now filter out OIDs that have already been linked to the project.

Relates to https://gitlab.com/gitlab-org/gitlab/-/issues/336047

Changelog: fixed

app/services/projects/lfs_pointers/lfs_object_download_list_service.rb

@@ -31,7 +31,7 @@ module Projects
         #
         LfsDownloadLinkListService
           .new(project, remote_uri: current_endpoint_uri)
-          .execute(lfs_pointers_in_repository)
+          .execute(missing_lfs_files)
       rescue LfsDownloadLinkListService::DownloadLinksError => e
         raise LfsObjectDownloadListError, "The LFS objects download list couldn't be imported. Error: #{e.message}"
       end
@@ -53,6 +53,22 @@ module Projects
         @lfs_pointers_in_repository ||= LfsListService.new(project).execute
       end
 
+      def existing_lfs_objects
+        project.lfs_objects
+      end
+
+      def existing_lfs_objects_hash
+        {}.tap do |hash|
+          existing_lfs_objects.find_each do |lfs_object|
+            hash[lfs_object.oid] = lfs_object.size
+          end
+        end
+      end
+
+      def missing_lfs_files
+        lfs_pointers_in_repository.except(*existing_lfs_objects_hash.keys)
+      end
+
       def lfsconfig_endpoint_uri
         strong_memoize(:lfsconfig_endpoint_uri) do
           # Retrieveing the blob data from the .lfsconfig file

spec/services/projects/lfs_pointers/lfs_object_download_list_service_spec.rb

@@ -34,10 +34,24 @@ RSpec.describe Projects::LfsPointers::LfsObjectDownloadListService do
         subject.execute
       end
 
-      it 'retrieves the download links of non existent objects' do
-        expect_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute).with(all_oids)
+      context 'when no LFS objects exist' do
+        before do
+          project.lfs_objects.delete_all
+        end
 
-        subject.execute
+        it 'retrieves all LFS objects' do
+          expect_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute).with(all_oids)
+
+          subject.execute
+        end
+      end
+
+      context 'when some LFS objects already exist' do
+        it 'retrieves the download links of non-existent objects' do
+          expect_any_instance_of(Projects::LfsPointers::LfsDownloadLinkListService).to receive(:execute).with(oids)
+
+          subject.execute
+        end
       end
     end