From 79cb4d99c0e47bfd988788ff38871e668367dfbf Mon Sep 17 00:00:00 2001
From: Bob Van Landuyt <bob@vanlanduyt.co>
Date: Fri, 30 Mar 2018 19:45:58 +0200
Subject: [PATCH] Import projects with LFS objects

If the LFS object already exist, we'll link it tot he existing one, if
not we'll create it.
---
 lib/gitlab/import_export/importer.rb          |  11 ++-
 lib/gitlab/import_export/lfs_restorer.rb      |  43 ++++++++++
 lib/gitlab/import_export/lfs_saver.rb         |   2 +-
 spec/fixtures/exported-project.gz             | Bin 0 -> 2306 bytes
 .../lib/gitlab/import_export/importer_spec.rb |  64 +++++++++++++++
 .../gitlab/import_export/lfs_restorer_spec.rb |  75 ++++++++++++++++++
 .../gitlab/import_export/lfs_saver_spec.rb    |   3 +-
 7 files changed, 195 insertions(+), 3 deletions(-)
 create mode 100644 lib/gitlab/import_export/lfs_restorer.rb
 create mode 100644 spec/fixtures/exported-project.gz
 create mode 100644 spec/lib/gitlab/import_export/importer_spec.rb
 create mode 100644 spec/lib/gitlab/import_export/lfs_restorer_spec.rb

diff --git a/lib/gitlab/import_export/importer.rb b/lib/gitlab/import_export/importer.rb
index c38df9102e..c490bf059d 100644
--- a/lib/gitlab/import_export/importer.rb
+++ b/lib/gitlab/import_export/importer.rb
@@ -13,7 +13,7 @@ module Gitlab
       end
 
       def execute
-        if import_file && check_version! && [repo_restorer, wiki_restorer, project_tree, avatar_restorer, uploads_restorer].all?(&:restore)
+        if import_file && check_version! && restorers.all?(&:restore)
           project_tree.restored_project
         else
           raise Projects::ImportService::Error.new(@shared.errors.join(', '))
@@ -24,6 +24,11 @@ module Gitlab
 
       private
 
+      def restorers
+        [repo_restorer, wiki_restorer, project_tree, avatar_restorer,
+         uploads_restorer, lfs_restorer]
+      end
+
       def import_file
         Gitlab::ImportExport::FileImporter.import(archive_file: @archive_file,
                                                   shared: @shared)
@@ -60,6 +65,10 @@ module Gitlab
         Gitlab::ImportExport::UploadsRestorer.new(project: project_tree.restored_project, shared: @shared)
       end
 
+      def lfs_restorer
+        Gitlab::ImportExport::LfsRestorer.new(project: project_tree.restored_project, shared: @shared)
+      end
+
       def path_with_namespace
         File.join(@project.namespace.full_path, @project.path)
       end
diff --git a/lib/gitlab/import_export/lfs_restorer.rb b/lib/gitlab/import_export/lfs_restorer.rb
new file mode 100644
index 0000000000..4f144d5c8e
--- /dev/null
+++ b/lib/gitlab/import_export/lfs_restorer.rb
@@ -0,0 +1,43 @@
+module Gitlab
+  module ImportExport
+    class LfsRestorer
+      def initialize(project:, shared:)
+        @project = project
+        @shared = shared
+      end
+
+      def restore
+        return true if lfs_file_paths.empty?
+
+        lfs_file_paths.each do |file_path|
+          link_or_create_lfs_object!(file_path)
+        end
+
+        true
+      rescue => e
+        @shared.error(e)
+        false
+      end
+
+      private
+
+      def link_or_create_lfs_object!(path)
+        size = File.size(path)
+        oid = LfsObject.calculate_oid(path)
+
+        lfs_object = LfsObject.find_or_initialize_by(oid: oid, size: size)
+        lfs_object.file = File.open(path) unless lfs_object.file&.exists?
+
+        @project.lfs_storage_project.lfs_objects << lfs_object
+      end
+
+      def lfs_file_paths
+        @lfs_file_paths ||= Dir.glob("#{lfs_storage_path}/*")
+      end
+
+      def lfs_storage_path
+        File.join(@shared.export_path, 'lfs-objects')
+      end
+    end
+  end
+end
diff --git a/lib/gitlab/import_export/lfs_saver.rb b/lib/gitlab/import_export/lfs_saver.rb
index bb7a070fe1..d796440902 100644
--- a/lib/gitlab/import_export/lfs_saver.rb
+++ b/lib/gitlab/import_export/lfs_saver.rb
@@ -11,7 +11,7 @@ module Gitlab
       def save
         return true if @project.lfs_objects.empty?
 
-        @project.lfs_objects.each do |lfs_object|
+        @project.lfs_storage_project.lfs_objects.each do |lfs_object|
           save_lfs_object(lfs_object)
         end
 
diff --git a/spec/fixtures/exported-project.gz b/spec/fixtures/exported-project.gz
new file mode 100644
index 0000000000000000000000000000000000000000..352384f16c83737e4c4bdfb55b8d413e0aeb3208
GIT binary patch
literal 2306
zcmV+d3H|mTiwFS9YQtIp1MOH1a23@R4zU6ah$#hHrX78({22O@?BDwfMFc?zKMf{<
zX^FV*+r96-OZM$P_9rC86bVLJs-ht@1tLRJT96JZwX}#uR3<_Q4nG0n2ny7JLUlT$
z43#NA_Uyjh<h@5Eai)Rk^qom&-`#ueJ>NOsIcM)~UVcWe3rUhbl5|;DugAKQG#kI#
z;-VPZOM1N?hN4|0MNtfs;qvr8w2TZx(|Hx31S_PQ>VaA)bd!18kB;mAh~(wRRAm7Y
z^t=U{BKM9Bh=-SDzmk6s%hD<QGad@~Cwm7@vizU(znBn_Ca5^3V?`zkWJ8kNL<QDx
z5KCCE;v`goB%*+H6TB)!a0L>Hf)Foh$V~{C!=kf9i|{HEIn|J5q-q?J`JiO#>Z$>%
zK?6%7Cnyz2<->?mQ5aP^E8}y#Bq@uyFxI&c(uGL!LLEnuV(3Hx<0Sz}49Oyv!<>jD
zq@xZzF(W8pfwPuq&IS^%>0mFWXjoU&Do%(Xp_~ix*g_XaASklVW58_!!<r6iDfksx
zs<I(QkqRE*5EsKSB*7|9GXw!4)1(a~!b1Qmb)?F?#9>V{5T~oWP>$3N6%{Qs5Fq1W
zsg@9~u69_2%W-1R0d5F~xfriU91ps}2SZN8nqX)eP^)pk5BR4kQX+>j70bFA%lI+}
z7Wh@38Ux<Y6;9w~0ZE+0fD;4oop99MM4T56sj8wn2yM$8&?xdcWQvGGp+tNgkdlyS
z9B5hIrG_X)O+`UoLpNwGP=XqO2G%(&i=9aqgEZ8*h_1&pYx|A`3Dp48QAmA)FtaF#
zB!E~p9}Gk4Xu4uKZH_I1&EJa&vm7W7?Iv`rOOTkvMA>Cc6IS>lifUC1WyalVy4KJW
z09sXKMTLhXCuoxL=aOtL$&^x5frl!fsXUsd=D{vTOze?fQc)A7iVIc76jg^A=woKh
z5DIW;v4w3&&ep-IgtE4BO+=6*ywn*ZNNCa8C9V{Ox`rZE64pryR2fveme5tqD&`<`
zIBKmW&7_cc9eN69m8Ydjkv_)9(hSYmwTndTIBaj7a5o8SXEJf7`GoqG6fJ2B=xMT^
zK$({c+(=0>VepU$qxhu7aJPi9JtWN5v-j7lVkS9-PSOZ=RI2C4<0mO$JMCmecQVk*
zPOD}UQ<CE$7GW91wag7P8OLHsx0N9H2uj8tIo?N6GzheH#D=E*CJd`f+0OzNn9~$P
z6;Ma4?4gZ*c6=|>dku9&5CG#47I>?)?5&ua29gax!_NzJ8^|O*XFoeNz%Z68u-T5H
z(Da=Qu<z+gIg(TEna;+NQ_k)($vIRR!_d?c?wHOJ1$Ih-Lfjs~i%~2;h${JLOhS2r
z61C$Z^HDQcs-oybwMmZ@f^G-R#mZi%*(7y2o2t&HYYuf4hr@Bo5+*qfdY))VjusfK
zz=AW1C`x%J@j8%YMofj~9!Y3rT@-~4Itn!i&fdHX$O4jte&p@03=;L?{wC&IDx4&+
zoPGbs1nnLeL-!1rsb|s?GqlJpLKf&DvVe)KI*CM`52sXg2+SEOax&B7zme6HXK6BK
znslCVT}+r~lP+#Mou-Sa`0=cZiEbMKOR+s_1h!tnp)b`~HCTglzY>K_pnQUcp4tY3
zyco6{=j}eB_papSL)*z!f~MD6FMSL6`tSb?!+O*2|F=T&^5;$|E-5VfQ7_u)=KbIN
z1dzZ#1^m-A=}W)=-wJ`pNFIH=L8RrjN$2;!pdpJA>J1y+)IYpWF)96@CDZkPYvevx
z1?>uB-4&-l%kTGld7luX=};&jA`0jU_<}qgBpJWJq5zzvyxssSxKtF<@*{{Bwfy+o
zK+l&>n=)~7R@Ur^56u7onegAQVNhk`i|=jAdWXz@^33?6g>Sencp6HxAL+MAuDLQ_
zeS6Z0X@z&5`lv=ZefjQtm&`9+nz`z}<>I(aN8dfo;E@*&l?`cmta#6~$YXy!|LBN^
zH_rX==}G?k(W%!r4BxdfXM5A;4_>{TJ@A!FKN+@T*wTIn4_$0LUR3+rA%Wdh_bl19
z^6WS68qxQIpAD)Qw$-z3X4~vPTyJ@A)Tr~yqJ4WdpBpgx{>g2t2LQZQ3tq;d@f^VW
zOl@af!De}<f5({De~{%}T9<Ww*Rmz2FF)NrY|gZ?>z|sm;hVV?c_nklJom?`s3G!?
z(XG{wf4lu(n+G&5s9Efe{bI`TqGi=vE?j=@?4^s|_B%fP?qO#6dR{m@Z`CJ1oKV&_
z;f>*wPkRUb{gcR>TkE}}Ug}rb$Uspr)FX#x9CR1Db8`029GhF18;V?dzhuX+C(!(f
z7q4BJuzu&onUhwO?9FbX_Z>Uj+&(AYS9|8*z@m>odT9UdXRcfgPp^GoZSlq}xbK|$
zRX?s@zh&L8)~wpnP`_sHTRDqQ{W<V*pdkEtb6G9ZHssN(Lu&fGt>(I4yQ8wPwa`7i
z=-8~*1BJ(C>_1SFK~FFKrKBBMF|?R&JMz{uCrdXBCTIL*<o2eg24tN7_uz%QU`@%~
z{w@35b2FMI*VY~V+u+)!qcu;|{^=@Fd*Ecv)uzGMj%H;yH=k|b>RwpW@bc`sS7tY^
z-SYek^Xm8Cnep?Sq2JpLF5lJM`mJ|d5c6?!4n5f?^V3N?&%D;389V$t+N(dUZdi19
z-QFi!-aOv2y!eZ4^2&Ele0ja4Y)w(~n4!jSYU=8fZT~z&KKOX`zLrhr);%2UUw(Md
z_GKTA+PW)ec;?8dBOZLVZ}!&K)xl@q&&;}qt2@5p-r~B^-~U}^N1*$whW?pxtaTQD
znc35}<BR7%&;R&=FV__RX65gPytH6#pNtcC54?6{<JNkA#?o15bEjWw_v{~cYW%sf
z+g;LMOOnt3loxp!k3YzJ_>d<Ar*;bYM1f>|B<rPF9u8`h5cEQsvj~b^3{YO5k70u%
z3Wxz;$nWdgMz`lbmP+pbG~**Z>HdE!q%?vx7yQYJ?&1aA;3cWb#k(Z)bEM1s8g;t?
crX?+DNlRMNl9sfj<-aQb0dgf7K>#QK0J+D5UjP6A

literal 0
HcmV?d00001

diff --git a/spec/lib/gitlab/import_export/importer_spec.rb b/spec/lib/gitlab/import_export/importer_spec.rb
new file mode 100644
index 0000000000..d75416f2a6
--- /dev/null
+++ b/spec/lib/gitlab/import_export/importer_spec.rb
@@ -0,0 +1,64 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::Importer do
+  let(:test_path) { "#{Dir.tmpdir}/importer_spec" }
+  let(:shared) { project.import_export_shared }
+  let(:project) { create(:project, import_source: File.join(test_path, 'exported-project.gz')) }
+
+  subject(:importer) { described_class.new(project) }
+
+  before do
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(test_path)
+    FileUtils.mkdir_p(shared.export_path)
+    FileUtils.cp(Rails.root.join('spec', 'fixtures', 'exported-project.gz'), test_path)
+  end
+
+  after do
+    FileUtils.rm_rf(test_path)
+  end
+
+  describe '#execute' do
+    it 'succeeds' do
+      importer.execute
+
+      expect(shared.errors).to be_empty
+    end
+
+    it 'extracts the archive'  do
+      expect(Gitlab::ImportExport::FileImporter).to receive(:import).and_call_original
+
+      importer.execute
+    end
+
+    it 'checks the version' do
+      expect(Gitlab::ImportExport::VersionChecker).to receive(:check!).and_call_original
+
+      importer.execute
+    end
+
+    context 'all restores are executed' do
+      [
+        Gitlab::ImportExport::AvatarRestorer,
+        Gitlab::ImportExport::RepoRestorer,
+        Gitlab::ImportExport::WikiRestorer,
+        Gitlab::ImportExport::UploadsRestorer,
+        Gitlab::ImportExport::LfsRestorer
+      ].each do |restorer|
+        it "calls the #{restorer}" do
+          fake_restorer = double(restorer.to_s)
+
+          expect(fake_restorer).to receive(:restore).and_return(true).at_least(1)
+          expect(restorer).to receive(:new).and_return(fake_restorer).at_least(1)
+
+          importer.execute
+        end
+      end
+
+      it 'restores the ProjectTree' do
+        expect(Gitlab::ImportExport::ProjectTreeRestorer).to receive(:new).and_call_original
+
+        importer.execute
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/import_export/lfs_restorer_spec.rb b/spec/lib/gitlab/import_export/lfs_restorer_spec.rb
new file mode 100644
index 0000000000..70eeb9ee66
--- /dev/null
+++ b/spec/lib/gitlab/import_export/lfs_restorer_spec.rb
@@ -0,0 +1,75 @@
+require 'spec_helper'
+
+describe Gitlab::ImportExport::LfsRestorer do
+  include UploadHelpers
+
+  let(:export_path) { "#{Dir.tmpdir}/lfs_object_restorer_spec" }
+  let(:project) { create(:project) }
+  let(:shared) { project.import_export_shared }
+  subject(:restorer) { described_class.new(project: project, shared: shared) }
+
+  before do
+    allow_any_instance_of(Gitlab::ImportExport).to receive(:storage_path).and_return(export_path)
+    FileUtils.mkdir_p(shared.export_path)
+  end
+
+  after do
+    FileUtils.rm_rf(shared.export_path)
+  end
+
+  describe '#restore' do
+    context 'when the archive contains lfs files' do
+      let(:dummy_lfs_file_path) { File.join(shared.export_path, 'lfs-objects', 'dummy') }
+
+      def create_lfs_object_with_content(content)
+        dummy_lfs_file = Tempfile.new('existing')
+        File.write(dummy_lfs_file.path, content)
+        size = dummy_lfs_file.size
+        oid = LfsObject.calculate_oid(dummy_lfs_file.path)
+        LfsObject.create!(oid: oid, size: size, file: dummy_lfs_file)
+      end
+
+      before do
+        FileUtils.mkdir_p(File.dirname(dummy_lfs_file_path))
+        File.write(dummy_lfs_file_path, 'not very large')
+        allow(restorer).to receive(:lfs_file_paths).and_return([dummy_lfs_file_path])
+      end
+
+      it 'creates an lfs object for the project' do
+        expect { restorer.restore }.to change { project.reload.lfs_objects.size }.by(1)
+      end
+
+      it 'assigns the file correctly' do
+        restorer.restore
+
+        expect(project.lfs_objects.first.file.read).to eq('not very large')
+      end
+
+      it 'links an existing LFS object if it existed' do
+        lfs_object = create_lfs_object_with_content('not very large')
+
+        restorer.restore
+
+        expect(project.lfs_objects).to include(lfs_object)
+      end
+
+      it 'succeeds' do
+        expect(restorer.restore).to be_truthy
+        expect(shared.errors).to be_empty
+      end
+
+      it 'stores the upload' do
+        expect_any_instance_of(LfsObjectUploader).to receive(:store!)
+
+        restorer.restore
+      end
+    end
+
+    context 'without any LFS-objects' do
+      it 'succeeds' do
+        expect(restorer.restore).to be_truthy
+        expect(shared.errors).to be_empty
+      end
+    end
+  end
+end
diff --git a/spec/lib/gitlab/import_export/lfs_saver_spec.rb b/spec/lib/gitlab/import_export/lfs_saver_spec.rb
index e2237cd22c..e62afac1c4 100644
--- a/spec/lib/gitlab/import_export/lfs_saver_spec.rb
+++ b/spec/lib/gitlab/import_export/lfs_saver_spec.rb
@@ -19,8 +19,9 @@ describe Gitlab::ImportExport::LfsSaver do
   describe '#save' do
     context 'when the project has LFS objects' do
       let(:lfs_object) { create(:lfs_object, :with_file) }
+
       before do
-        project.lfs_objects << lfs_object\
+        project.lfs_objects << lfs_object
       end
 
       it 'does not cause errors' do
-- 
2.30.9