Commit 4223a6fe authored by James Lopez's avatar James Lopez

refactor uploads manager to grab uploads in batches

parent 8ea2027f
...@@ -3,6 +3,8 @@ module Gitlab ...@@ -3,6 +3,8 @@ module Gitlab
class UploadsManager class UploadsManager
include Gitlab::ImportExport::CommandLineUtil include Gitlab::ImportExport::CommandLineUtil
UPLOADS_BATCH_SIZE = 100
def initialize(project:, shared:, relative_export_path: 'uploads', from: nil) def initialize(project:, shared:, relative_export_path: 'uploads', from: nil)
@project = project @project = project
@shared = shared @shared = shared
...@@ -54,7 +56,7 @@ module Gitlab ...@@ -54,7 +56,7 @@ module Gitlab
def copy_from_object_storage def copy_from_object_storage
return unless Gitlab::ImportExport.object_storage? return unless Gitlab::ImportExport.object_storage?
uploads.each do |upload_model| uploads do |upload_model|
next unless upload_model.file next unless upload_model.file
next if upload_model.upload.local? # Already copied, using the old method next if upload_model.upload.local? # Already copied, using the old method
...@@ -71,15 +73,23 @@ module Gitlab ...@@ -71,15 +73,23 @@ module Gitlab
end end
def uploads def uploads
@uploads ||= begin avatar_path = @project.avatar&.upload&.path
if @relative_export_path == 'avatar'
[@project.avatar].compact if @relative_export_path == 'avatar'
else yield(@project.avatar)
(@project.uploads - [@project.avatar&.upload]).map(&:build_uploader) else
project_uploads(avatar_path).find_each(batch_size: UPLOADS_BATCH_SIZE) do |upload|
yield(upload.build_uploader)
end end
end end
end end
def project_uploads(avatar_path)
return @project.uploads unless avatar_path
@project.uploads.where("path != ?", avatar_path)
end
def download_and_copy(upload) def download_and_copy(upload)
secret = upload.try(:secret) || '' secret = upload.try(:secret) || ''
upload_path = File.join(uploads_export_path, secret, upload.filename) upload_path = File.join(uploads_export_path, secret, upload.filename)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment