Commit 32d2206b authored by Douwe Maan's avatar Douwe Maan

Merge branch 'bvl-export-import-lfs' into 'master'

Export and import LFS objects

Closes #40643

See merge request gitlab-org/gitlab-ce!18115
parents 4ef3e835 48b17e99
...@@ -41,7 +41,7 @@ class Projects::LfsApiController < Projects::GitHttpClientController ...@@ -41,7 +41,7 @@ class Projects::LfsApiController < Projects::GitHttpClientController
def existing_oids def existing_oids
@existing_oids ||= begin @existing_oids ||= begin
storage_project.lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid) project.all_lfs_objects.where(oid: objects.map { |o| o['oid'].to_s }).pluck(:oid)
end end
end end
......
...@@ -1066,6 +1066,16 @@ class Project < ActiveRecord::Base ...@@ -1066,6 +1066,16 @@ class Project < ActiveRecord::Base
end end
end end
# This will return all `lfs_objects` that are accessible to the project.
# So this might be `self.lfs_objects` if the project is not part of a fork
# network, or it is the base of the fork network.
#
# TODO: refactor this to get the correct lfs objects when implementing
# https://gitlab.com/gitlab-org/gitlab-ce/issues/39769
def all_lfs_objects
lfs_storage_project.lfs_objects
end
def personal? def personal?
!group !group
end end
......
...@@ -28,7 +28,7 @@ module Projects ...@@ -28,7 +28,7 @@ module Projects
end end
def save_services def save_services
[version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver].all?(&:save) [version_saver, avatar_saver, project_tree_saver, uploads_saver, repo_saver, wiki_repo_saver, lfs_saver].all?(&:save)
end end
def version_saver def version_saver
...@@ -55,6 +55,10 @@ module Projects ...@@ -55,6 +55,10 @@ module Projects
Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared) Gitlab::ImportExport::WikiRepoSaver.new(project: project, shared: @shared)
end end
def lfs_saver
Gitlab::ImportExport::LfsSaver.new(project: project, shared: @shared)
end
def cleanup_and_notify_error def cleanup_and_notify_error
Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}") Rails.logger.error("Import/Export - Project #{project.name} with ID: #{project.id} export error - #{@shared.errors.join(', ')}")
......
...@@ -21,11 +21,11 @@ ...@@ -21,11 +21,11 @@
%li Project uploads %li Project uploads
%li Project configuration including web hooks and services %li Project configuration including web hooks and services
%li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities %li Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, and other project entities
%li LFS objects
%p %p
The following items will NOT be exported: The following items will NOT be exported:
%ul %ul
%li Job traces and artifacts %li Job traces and artifacts
%li LFS objects
%li Container registry images %li Container registry images
%li CI variables %li CI variables
%li Any encrypted tokens %li Any encrypted tokens
......
---
title: Support LFS objects when importing/exporting GitLab project archives
merge_request: 18115
author:
type: added
...@@ -57,11 +57,11 @@ The following items will be exported: ...@@ -57,11 +57,11 @@ The following items will be exported:
- Project configuration including web hooks and services - Project configuration including web hooks and services
- Issues with comments, merge requests with diffs and comments, labels, milestones, snippets, - Issues with comments, merge requests with diffs and comments, labels, milestones, snippets,
and other project entities and other project entities
- LFS objects
The following items will NOT be exported: The following items will NOT be exported:
- Build traces and artifacts - Build traces and artifacts
- LFS objects
- Container registry images - Container registry images
- CI variables - CI variables
- Any encrypted tokens - Any encrypted tokens
......
...@@ -15,8 +15,7 @@ module Gitlab ...@@ -15,8 +15,7 @@ module Gitlab
return false unless new_lfs_pointers.present? return false unless new_lfs_pointers.present?
existing_count = @project.lfs_storage_project existing_count = @project.all_lfs_objects
.lfs_objects
.where(oid: new_lfs_pointers.map(&:lfs_oid)) .where(oid: new_lfs_pointers.map(&:lfs_oid))
.count .count
......
...@@ -13,7 +13,7 @@ module Gitlab ...@@ -13,7 +13,7 @@ module Gitlab
end end
def execute def execute
if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore) if import_file && check_version! && restorers.all?(&:restore)
project_tree.restored_project project_tree.restored_project
else else
raise Projects::ImportService::Error.new(@shared.errors.join(', ')) raise Projects::ImportService::Error.new(@shared.errors.join(', '))
...@@ -24,6 +24,11 @@ module Gitlab ...@@ -24,6 +24,11 @@ module Gitlab
private private
def restorers
[repo_restorer, wiki_restorer, project_tree, avatar_restorer,
uploads_restorer, lfs_restorer]
end
def import_file def import_file
Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file, Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
shared: @shared) shared: @shared)
...@@ -60,6 +65,10 @@ module Gitlab ...@@ -60,6 +65,10 @@ module Gitlab
Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared) Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
end end
def lfs_restorer
Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
end
def path_with_namespace def path_with_namespace
File.join(@project.namespace.full_path, @project.path) File.join(@project.namespace.full_path, @project.path)
end end
......
module Gitlab
module ImportExport
class LfsRestorer
def initialize(project:, shared:)
@project = project
@shared = shared
end
def restore
return true if lfs_file_paths.empty?
lfs_file_paths.each do |file_path|
link_or_create_lfs_object!(file_path)
end
true
rescue => e
@shared.error(e)
false
end
private
def link_or_create_lfs_object!(path)
size = File.size(path)
oid = LfsObject.calculate_oid(path)
lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size)
lfs_object.file = File.open(path) unless lfs_object.file&.exists?
@project.all_lfs_objects << lfs_object
end
def lfs_file_paths
@lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*")
end
def lfs_storage_path
File.join(@shared.export_path, 'lfs-objects')
end
end
end
end
module Gitlab
module ImportExport
class LfsSaver
include Gitlab::ImportExport::CommandLineUtil
def initialize(project:, shared:)
@project = project
@shared = shared
end
def save
@project.all_lfs_objects.each do |lfs_object|
save_lfs_object(lfs_object)
end
true
rescue => e
@shared.error(e)
false
end
private
def save_lfs_object(lfs_object)
if lfs_object.local_store?
copy_file_for_lfs_object(lfs_object)
else
download_file_for_lfs_object(lfs_object)
end
end
def download_file_for_lfs_object(lfs_object)
destination = destination_path_for_object(lfs_object)
mkdir_p(File.dirname(destination))
File.open(destination, 'w') do |file|
IO.copy_stream(URI.parse(lfs_object.file.url).open, file)
end
end
def copy_file_for_lfs_object(lfs_object)
copy_files(lfs_object.file.path, destination_path_for_object(lfs_object))
end
def destination_path_for_object(lfs_object)
File.join(lfs_export_path, lfs_object.oid)
end
def lfs_export_path
File.join(@shared.export_path, 'lfs-objects')
end
end
end
end
require 'spec_helper'
describe Gitlab::ImportExport::Importer do
let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
let(:shared) { project.import_export_shared }
let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) }
subject(:importer) { described_class.new(project) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
FileUtils.mkdir_p(shared.export_path)
FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path)
end
after do
FileUtils.rm_rf(test_path)
end
describe '#execute' do
it 'succeeds' do
importer.execute
expect(shared.errors).to be_empty
end
it 'extracts the archive' do
expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
importer.execute
end
it 'checks the version' do
expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
importer.execute
end
context 'all restores are executed' do
[
Gitlab::ImportExport::AvatarRestorer,
Gitlab::ImportExport::RepoRestorer,
Gitlab::ImportExport::WikiRestorer,
Gitlab::ImportExport::UploadsRestorer,
Gitlab::ImportExport::LfsRestorer
].each do |restorer|
it "calls the #{restorer}" do
fake_restorer = double(restorer.to_s)
expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
importer.execute
end
end
it 'restores the ProjectTree' do
expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
importer.execute
end
end
end
end
require 'spec_helper'
describe Gitlab::ImportExport::LfsRestorer do
include UploadHelpers
let(:export_path) { "#{Dir.tmpdir}/lfs_object_restorer_spec" }
let(:project) { create(:project) }
let(:shared) { project.import_export_shared }
subject(:restorer) { described_class.new(project: project, shared: shared) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
FileUtils.mkdir_p(shared.export_path)
end
after do
FileUtils.rm_rf(shared.export_path)
end
describe '#restore' do
context 'when the archive contains lfs files' do
let(:dummy_lfs_file_path) { File.join(shared.export_path, 'lfs-objects', 'dummy') }
def create_lfs_object_with_content(content)
dummy_lfs_file = Tempfile.new('existing')
File.write(dummy_lfs_file.path, content)
size = dummy_lfs_file.size
oid = LfsObject.calculate_oid(dummy_lfs_file.path)
LfsObject.create!(oid: oid, size: size, file: dummy_lfs_file)
end
before do
FileUtils.mkdir_p(File.dirname(dummy_lfs_file_path))
File.write(dummy_lfs_file_path, 'not very large')
allow(restorer).to receive(:lfs_file_paths).and_return([dummy_lfs_file_path])
end
it 'creates an lfs object for the project' do
expect { restorer.restore }.to change { project.reload.lfs_objects.size }.by(1)
end
it 'assigns the file correctly' do
restorer.restore
expect(project.lfs_objects.first.file.read).to eq('not very large')
end
it 'links an existing LFS object if it existed' do
lfs_object = create_lfs_object_with_content('not very large')
restorer.restore
expect(project.lfs_objects).to include(lfs_object)
end
it 'succeeds' do
expect(restorer.restore).to be_truthy
expect(shared.errors).to be_empty
end
it 'stores the upload' do
expect_any_instance_of(LfsObjectUploader).to receive(:store!)
restorer.restore
end
end
context 'without any LFS-objects' do
it 'succeeds' do
expect(restorer.restore).to be_truthy
expect(shared.errors).to be_empty
end
end
end
end
require 'spec_helper'
describe Gitlab::ImportExport::LfsSaver do
let(:shared) { project.import_export_shared }
let(:export_path) { "#{Dir.tmpdir}/project_tree_saver_spec" }
let(:project) { create(:project) }
subject(:saver) { described_class.new(project: project, shared: shared) }
before do
allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
FileUtils.mkdir_p(shared.export_path)
end
after do
FileUtils.rm_rf(shared.export_path)
end
describe '#save' do
context 'when the project has LFS objects locally stored' do
let(:lfs_object) { create(:lfs_object, :with_file) }
before do
project.lfs_objects << lfs_object
end
it 'does not cause errors' do
saver.save
expect(shared.errors).to be_empty
end
it 'copies the file in the correct location when there is an lfs object' do
saver.save
expect(File).to exist("#{shared.export_path}/lfs-objects/#{lfs_object.oid}")
end
end
context 'when the LFS objects are stored in object storage' do
let(:lfs_object) { create(:lfs_object, :object_storage) }
before do
allow(LfsObjectUploader).to receive(:object_store_enabled?).and_return(true)
allow(lfs_object.file).to receive(:url).and_return('http://my-object-storage.local')
project.lfs_objects << lfs_object
end
it 'downloads the file to include in an archive' do
fake_uri = double
exported_file_path = "#{shared.export_path}/lfs-objects/#{lfs_object.oid}"
expect(fake_uri).to receive(:open).and_return(StringIO.new('LFS file content'))
expect(URI).to receive(:parse).with('http://my-object-storage.local').and_return(fake_uri)
saver.save
expect(File.read(exported_file_path)).to eq('LFS file content')
end
end
end
end
...@@ -2022,6 +2022,22 @@ describe Project do ...@@ -2022,6 +2022,22 @@ describe Project do
expect(forked_project.lfs_storage_project).to eq forked_project expect(forked_project.lfs_storage_project).to eq forked_project
end end
end end
describe '#all_lfs_objects' do
let(:lfs_object) { create(:lfs_object) }
before do
project.lfs_objects << lfs_object
end
it 'returns the lfs object for a project' do
expect(project.all_lfs_objects).to contain_exactly(lfs_object)
end
it 'returns the lfs object for a fork' do
expect(forked_project.all_lfs_objects).to contain_exactly(lfs_object)
end
end
end end
describe '#pushes_since_gc' do describe '#pushes_since_gc' do
......
...@@ -8,6 +8,49 @@ describe Projects::ImportExport::ExportService do ...@@ -8,6 +8,49 @@ describe Projects::ImportExport::ExportService do
let(:service) { described_class.new(project, user) } let(:service) { described_class.new(project, user) }
let!(:after_export_strategy) { Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy.new } let!(:after_export_strategy) { Gitlab::ImportExport::AfterExportStrategies::DownloadNotificationStrategy.new }
it 'saves the version' do
expect(Gitlab::ImportExport::VersionSaver).to receive(:new).and_call_original
service.execute
end
it 'saves the avatar' do
expect(Gitlab::ImportExport::AvatarSaver).to receive(:new).and_call_original
service.execute
end
it 'saves the models' do
expect(Gitlab::ImportExport::ProjectTreeSaver).to receive(:new).and_call_original
service.execute
end
it 'saves the uploads' do
expect(Gitlab::ImportExport::UploadsSaver).to receive(:new).and_call_original
service.execute
end
it 'saves the repo' do
# once for the normal repo, once for the wiki
expect(Gitlab::ImportExport::RepoSaver).to receive(:new).twice.and_call_original
service.execute
end
it 'saves the lfs objects' do
expect(Gitlab::ImportExport::LfsSaver).to receive(:new).and_call_original
service.execute
end
it 'saves the wiki repo' do
expect(Gitlab::ImportExport::WikiRepoSaver).to receive(:new).and_call_original
service.execute
end
context 'when all saver services succeed' do context 'when all saver services succeed' do
before do before do
allow(service).to receive(:save_services).and_return(true) allow(service).to receive(:save_services).and_return(true)
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment