using shared path for project import uploads and refactored gitlab remove export worker

CHANGELOG

@@ -47,6 +47,9 @@ v 8.11.0 (unreleased)
   - Fix RequestProfiler::Middleware error when code is reloaded in development
   - Catch what warden might throw when profiling requests to re-throw it
 
+v 8.10.4 (unreleased)
+  - Fix Import/Export project import not working in HA mode
+
 v 8.10.3
   - Fix Import/Export issue importing milestones and labels not associated properly. !5426
   - Fix timing problems running imports on production. !5523

app/controllers/import/gitlab_projects_controller.rb

@@ -12,13 +12,14 @@ class Import::GitlabProjectsController < Import::BaseController
       return redirect_back_or_default(options: { alert: "You need to upload a GitLab project export archive." })
     end
 
-    imported_file = project_params[:file].path + "-import"
+    import_upload_path = Gitlab::ImportExport.import_upload_path(filename: project_params[:file].original_filename)
 
-    FileUtils.copy_entry(project_params[:file].path, imported_file)
+    FileUtils.mkdir_p(File.dirname(import_upload_path))
+    FileUtils.copy_entry(project_params[:file].path, import_upload_path)
 
     @project = Gitlab::ImportExport::ProjectCreator.new(project_params[:namespace_id],
                                                         current_user,
-                                                        File.expand_path(imported_file),
+                                                        import_upload_path,
                                                         project_params[:path]).execute
 
     if @project.saved?

app/models/project.rb

@@ -378,11 +378,6 @@ class Project < ActiveRecord::Base
 
       joins(join_body).reorder('join_note_counts.amount DESC')
     end
-
-    # Deletes gitlab project export files older than 24 hours
-    def remove_gitlab_exports!
-      Gitlab::Popen.popen(%W(find #{Gitlab::ImportExport.storage_path} -not -path #{Gitlab::ImportExport.storage_path} -mmin +1440 -delete))
-    end
   end
 
   def repository_storage_path

app/services/import_export_clean_up_service.rb

+class ImportExportCleanUpService
+  LAST_MODIFIED_TIME_IN_MINUTES = 1440
+
+  attr_reader :mmin, :path
+
+  def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
+    @mmin = mmin
+    @path = Gitlab::ImportExport.storage_path
+  end
+
+  def execute
+    Gitlab::Metrics.measure(:import_export_clean_up) do
+      return unless File.directory?(path)
+
+      clean_up_export_files
+    end
+  end
+
+  private
+
+  def clean_up_export_files
+    Gitlab::Popen.popen(%W(find #{path} -not -path #{path} -mmin +#{mmin} -delete))
+  end
+end

app/services/repository_archive_clean_up_service.rb

 class RepositoryArchiveCleanUpService
   LAST_MODIFIED_TIME_IN_MINUTES = 120
 
+  attr_reader :mmin, :path
+
   def initialize(mmin = LAST_MODIFIED_TIME_IN_MINUTES)
     @mmin = mmin
     @path = Gitlab.config.gitlab.repository_downloads_path
@@ -17,8 +19,6 @@ class RepositoryArchiveCleanUpService
 
   private
 
-  attr_reader :mmin, :path
-
   def clean_up_old_archives
     run(%W(find #{path} -not -path #{path} -type f \( -name \*.tar -o -name \*.bz2 -o -name \*.tar.gz -o -name \*.zip \) -maxdepth 2 -mmin +#{mmin} -delete))
   end

app/workers/gitlab_remove_project_export_worker.rb → app/workers/import_export_project_cleanup_worker.rb

-class GitlabRemoveProjectExportWorker
+class ImportExportProjectCleanupWorker
   include Sidekiq::Worker
 
   sidekiq_options queue: :default
 
   def perform
-    Project.remove_gitlab_exports!
+    ImportExportCleanUpService.new.execute
   end
 end

config/initializers/1_settings.rb

@@ -287,9 +287,9 @@ Settings.cron_jobs['admin_email_worker']['job_class'] = 'AdminEmailWorker'
 Settings.cron_jobs['repository_archive_cache_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['repository_archive_cache_worker']['cron'] ||= '0 * * * *'
 Settings.cron_jobs['repository_archive_cache_worker']['job_class'] = 'RepositoryArchiveCacheWorker'
-Settings.cron_jobs['gitlab_remove_project_export_worker'] ||= Settingslogic.new({})
-Settings.cron_jobs['gitlab_remove_project_export_worker']['cron'] ||= '0 * * * *'
-Settings.cron_jobs['gitlab_remove_project_export_worker']['job_class'] = 'GitlabRemoveProjectExportWorker'
+Settings.cron_jobs['import_export_project_cleanup_worker'] ||= Settingslogic.new({})
+Settings.cron_jobs['import_export_project_cleanup_worker']['cron'] ||= '0 * * * *'
+Settings.cron_jobs['import_export_project_cleanup_worker']['job_class'] = 'ImportExportProjectCleanupWorker'
 Settings.cron_jobs['requests_profiles_worker'] ||= Settingslogic.new({})
 Settings.cron_jobs['requests_profiles_worker']['cron'] ||= '0 0 * * *'
 Settings.cron_jobs['requests_profiles_worker']['job_class'] = 'RequestsProfilesWorker'

lib/gitlab/import_export.rb

@@ -13,6 +13,10 @@ module Gitlab
       File.join(Settings.shared['path'], 'tmp/project_exports')
     end
 
+    def import_upload_path(filename:)
+      File.join(storage_path, 'uploads', filename)
+    end
+
     def project_filename
       "project.json"
     end

spec/services/import_export_clean_up_service_spec.rb

+require 'spec_helper'
+
+describe ImportExportCleanUpService, services: true do
+  describe '#execute' do
+    let(:service) { described_class.new }
+
+    let(:tmp_import_export_folder) { 'tmp/project_exports' }
+
+    context 'when the import/export directory does not exist' do
+      it 'does not remove any archives' do
+        path = '/invalid/path/'
+        stub_repository_downloads_path(path)
+
+        expect(File).to receive(:directory?).with(path + tmp_import_export_folder).and_return(false).at_least(:once)
+        expect(service).not_to receive(:clean_up_export_files)
+
+        service.execute
+      end
+    end
+
+    context 'when the import/export directory exists' do
+      it 'removes old files' do
+        in_directory_with_files(mtime: 2.days.ago) do |dir, files|
+          service.execute
+
+          files.each { |file| expect(File.exist?(file)).to eq false }
+          expect(File.directory?(dir)).to eq false
+        end
+      end
+
+      it 'does not remove new files' do
+        in_directory_with_files(mtime: 2.hours.ago) do |dir, files|
+          service.execute
+
+          files.each { |file| expect(File.exist?(file)).to eq true }
+          expect(File.directory?(dir)).to eq true
+        end
+      end
+    end
+
+    def in_directory_with_files(mtime:)
+      Dir.mktmpdir do |tmpdir|
+        stub_repository_downloads_path(tmpdir)
+        dir = File.join(tmpdir, tmp_import_export_folder, 'subfolder')
+        FileUtils.mkdir_p(dir)
+
+        files = FileUtils.touch(file_list(dir) + [dir], mtime: mtime)
+
+        yield(dir, files)
+      end
+    end
+
+    def stub_repository_downloads_path(path)
+      new_shared_settings = Settings.shared.merge('path' => path)
+      allow(Settings).to receive(:shared).and_return(new_shared_settings)
+    end
+
+    def file_list(dir)
+      Array.new(5) do |num|
+        File.join(dir, "random-#{num}.tar.gz")
+      end
+    end
+  end
+end