Merge branch 'jh-group_import_ui_backend' into 'master'

Create endpoints for Group import UI

See merge request gitlab-org/gitlab!29270

app/controllers/concerns/workhorse_import_export_upload.rb

+# frozen_string_literal: true
+
+module WorkhorseImportExportUpload
+  extend ActiveSupport::Concern
+  include WorkhorseRequest
+
+  included do
+    skip_before_action :verify_authenticity_token, only: %i[authorize]
+    before_action :verify_workhorse_api!, only: %i[authorize]
+  end
+
+  def authorize
+    set_workhorse_internal_api_content_type
+
+    authorized = ImportExportUploader.workhorse_authorize(
+      has_length: false,
+      maximum_size: ImportExportUpload::MAXIMUM_IMPORT_FILE_SIZE
+    )
+
+    render json: authorized
+  rescue SocketError
+    render json: _("Error uploading file"), status: :internal_server_error
+  end
+
+  private
+
+  def file_is_valid?(file)
+    return false unless file.is_a?(::UploadedFile)
+
+    ImportExportUploader::EXTENSION_WHITELIST
+      .include?(File.extname(file.original_filename).delete('.'))
+  end
+end

app/controllers/import/gitlab_groups_controller.rb

+# frozen_string_literal: true
+
+class Import::GitlabGroupsController < ApplicationController
+  include WorkhorseImportExportUpload
+
+  before_action :ensure_group_import_enabled
+  before_action :import_rate_limit, only: %i[create]
+
+  def create
+    unless file_is_valid?(group_params[:file])
+      return redirect_back_or_default(options: { alert: _('Unable to process group import file') })
+    end
+
+    group_data = group_params.except(:file).merge(
+      visibility_level: closest_allowed_visibility_level,
+      import_export_upload: ImportExportUpload.new(import_file: group_params[:file])
+    )
+
+    group = ::Groups::CreateService.new(current_user, group_data).execute
+
+    if group.persisted?
+      Groups::ImportExport::ImportService.new(group: group, user: current_user).async_execute
+
+      redirect_to(
+        group_path(group),
+        notice: _("Group '%{group_name}' is being imported.") % { group_name: group.name }
+      )
+    else
+      redirect_back_or_default(
+        options: { alert: _("Group could not be imported: %{errors}") % { errors: group.errors.full_messages.to_sentence } }
+      )
+    end
+  end
+
+  private
+
+  def group_params
+    params.permit(:path, :name, :parent_id, :file)
+  end
+
+  def closest_allowed_visibility_level
+    if group_params[:parent_id].present?
+      parent_group = Group.find(group_params[:parent_id])
+
+      Gitlab::VisibilityLevel.closest_allowed_level(parent_group.visibility_level)
+    else
+      Gitlab::VisibilityLevel::PRIVATE
+    end
+  end
+
+  def ensure_group_import_enabled
+    render_404 unless Feature.enabled?(:group_import_export, @group, default_enabled: true)
+  end
+
+  def import_rate_limit
+    if Gitlab::ApplicationRateLimiter.throttled?(:group_import, scope: current_user)
+      Gitlab::ApplicationRateLimiter.log_request(request, :group_import_request_limit, current_user)
+
+      flash[:alert] = _('This endpoint has been requested too many times. Try again later.')
+      redirect_to new_group_path
+    end
+  end
+end

app/controllers/import/gitlab_projects_controller.rb

 # frozen_string_literal: true
 
 class Import::GitlabProjectsController < Import::BaseController
-  include WorkhorseRequest
+  include WorkhorseImportExportUpload
 
   before_action :whitelist_query_limiting, only: [:create]
   before_action :verify_gitlab_project_import_enabled
 
-  skip_before_action :verify_authenticity_token, only: [:authorize]
-  before_action :verify_workhorse_api!, only: [:authorize]
-
   def new
     @namespace = Namespace.find(project_params[:namespace_id])
     return render_404 unless current_user.can?(:create_projects, @namespace)
@@ -17,7 +14,7 @@ class Import::GitlabProjectsController < Import::BaseController
   end
 
   def create
-    unless file_is_valid?
+    unless file_is_valid?(project_params[:file])
       return redirect_back_or_default(options: { alert: _("You need to upload a GitLab project export archive (ending in .gz).") })
     end
 
@@ -33,28 +30,8 @@ class Import::GitlabProjectsController < Import::BaseController
     end
   end
 
-  def authorize
-    set_workhorse_internal_api_content_type
-
-    authorized = ImportExportUploader.workhorse_authorize(
-      has_length: false,
-      maximum_size: Gitlab::CurrentSettings.max_attachment_size.megabytes.to_i)
-
-    render json: authorized
-  rescue SocketError
-    render json: _("Error uploading file"), status: :internal_server_error
-  end
-
   private
 
-  def file_is_valid?
-    return false unless project_params[:file].is_a?(::UploadedFile)
-
-    filename = project_params[:file].original_filename
-
-    ImportExportUploader::EXTENSION_WHITELIST.include?(File.extname(filename).delete('.'))
-  end
-
   def verify_gitlab_project_import_enabled
     render_404 unless gitlab_project_import_enabled?
   end

app/models/import_export_upload.rb

@@ -4,6 +4,8 @@ class ImportExportUpload < ApplicationRecord
   include WithUploads
   include ObjectStorage::BackgroundMove
 
+  MAXIMUM_IMPORT_FILE_SIZE = 50.megabytes.freeze
+
   belongs_to :project
   belongs_to :group
 

app/services/groups/import_export/import_service.rb

@@ -11,6 +11,10 @@ module Groups
         @shared = Gitlab::ImportExport::Shared.new(@group)
       end
 
+      def async_execute
+        GroupImportWorker.perform_async(current_user.id, group.id)
+      end
+
       def execute
         if valid_user_permissions? && import_file && restorer.restore
           notify_success

config/routes/import.rb

@@ -63,6 +63,10 @@ namespace :import do
     post :authorize
   end
 
+  resource :gitlab_group, only: [:create] do
+    post :authorize
+  end
+
   resource :manifest, only: [:create, :new], controller: :manifest do
     get :status
     get :jobs

lib/api/group_import.rb

@@ -2,8 +2,6 @@
 
 module API
   class GroupImport < Grape::API
-    MAXIMUM_FILE_SIZE = 50.megabytes.freeze
-
     helpers do
       def parent_group
         find_group!(params[:parent_id]) if params[:parent_id].present?
@@ -38,7 +36,10 @@ module API
         status 200
         content_type Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
 
-        ImportExportUploader.workhorse_authorize(has_length: false, maximum_size: MAXIMUM_FILE_SIZE)
+        ImportExportUploader.workhorse_authorize(
+          has_length: false,
+          maximum_size: ImportExportUpload::MAXIMUM_IMPORT_FILE_SIZE
+        )
       end
 
       desc 'Create a new group import' do
@@ -76,7 +77,7 @@ module API
         group = ::Groups::CreateService.new(current_user, group_params).execute
 
         if group.persisted?
-          GroupImportWorker.perform_async(current_user.id, group.id) # rubocop:disable CodeReuse/Worker
+          ::Groups::ImportExport::ImportService.new(group: group, user: current_user).async_execute
 
           accepted!
         else

lib/gitlab/application_rate_limiter.rb

@@ -28,7 +28,8 @@ module Gitlab
           play_pipeline_schedule:      { threshold: 1, interval: 1.minute },
           show_raw_controller:         { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute },
           group_export:                { threshold: 1, interval: 5.minutes },
-          group_download_export:       { threshold: 10, interval: 10.minutes }
+          group_download_export:       { threshold: 10, interval: 10.minutes },
+          group_import:                { threshold: 30, interval: 5.minutes }
         }.freeze
       end
 

locale/gitlab.pot

@@ -10662,6 +10662,9 @@ msgstr ""
 msgid "Group %{group_name} was successfully created."
 msgstr ""
 
+msgid "Group '%{group_name}' is being imported."
+msgstr ""
+
 msgid "Group Audit Events"
 msgstr ""
 
@@ -10695,6 +10698,9 @@ msgstr ""
 msgid "Group avatar"
 msgstr ""
 
+msgid "Group could not be imported: %{errors}"
+msgstr ""
+
 msgid "Group description"
 msgstr ""
 
@@ -23104,6 +23110,9 @@ msgstr ""
 msgid "Unable to load the merge request widget. Try reloading the page."
 msgstr ""
 
+msgid "Unable to process group import file"
+msgstr ""
+
 msgid "Unable to resolve"
 msgstr ""
 

spec/requests/import/gitlab_groups_controller_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Import::GitlabGroupsController do
+  include WorkhorseHelpers
+
+  let(:import_path) { "#{Dir.tmpdir}/gitlab_groups_controller_spec" }
+  let(:workhorse_token) { JWT.encode({ 'iss' => 'gitlab-workhorse' }, Gitlab::Workhorse.secret, 'HS256') }
+  let(:workhorse_headers) do
+    { 'GitLab-Workhorse' => '1.0', Gitlab::Workhorse::INTERNAL_API_REQUEST_HEADER => workhorse_token }
+  end
+
+  before do
+    allow_next_instance_of(Gitlab::ImportExport) do |import_export|
+      expect(import_export).to receive(:storage_path).and_return(import_path)
+    end
+
+    stub_uploads_object_storage(ImportExportUploader)
+  end
+
+  after do
+    FileUtils.rm_rf(import_path, secure: true)
+  end
+
+  describe 'POST create' do
+    subject(:import_request) { upload_archive(file_upload, workhorse_headers, request_params) }
+
+    let_it_be(:user) { create(:user) }
+
+    let(:file) { File.join('spec', %w[fixtures group_export.tar.gz]) }
+    let(:file_upload) { fixture_file_upload(file) }
+
+    before do
+      login_as(user)
+    end
+
+    def upload_archive(file, headers = {}, params = {})
+      workhorse_finalize(
+        import_gitlab_group_path,
+        method: :post,
+        file_key: :file,
+        params: params.merge(file: file),
+        headers: headers,
+        send_rewritten_field: true
+      )
+    end
+
+    context 'when importing without a parent group' do
+      let(:request_params) { { path: 'test-group-import', name: 'test-group-import' } }
+
+      it 'successfully creates the group' do
+        expect { import_request }.to change { Group.count }.by 1
+
+        group = Group.find_by(name: 'test-group-import')
+
+        expect(response).to have_gitlab_http_status(:found)
+        expect(response).to redirect_to(group_path(group))
+        expect(flash[:notice]).to include('is being imported')
+      end
+
+      it 'imports the group data', :sidekiq_inline do
+        allow(GroupImportWorker).to receive(:perform_async).and_call_original
+
+        import_request
+
+        group = Group.find_by(name: 'test-group-import')
+
+        expect(GroupImportWorker).to have_received(:perform_async).with(user.id, group.id)
+
+        expect(group.description).to eq 'A voluptate non sequi temporibus quam at.'
+        expect(group.visibility_level).to eq Gitlab::VisibilityLevel::PRIVATE
+      end
+    end
+
+    context 'when importing to a parent group' do
+      let(:request_params) { { path: 'test-group-import', name: 'test-group-import', parent_id: parent_group.id } }
+      let(:parent_group) { create(:group) }
+
+      before do
+        parent_group.add_owner(user)
+      end
+
+      it 'creates a new group under the parent' do
+        expect { import_request }
+          .to change { parent_group.children.reload.size }.by 1
+
+        expect(response).to have_gitlab_http_status(:found)
+      end
+
+      shared_examples 'is created with the parent visibility level' do |visibility_level|
+        before do
+          parent_group.update!(visibility_level: visibility_level)
+        end
+
+        it "imports a #{Gitlab::VisibilityLevel.level_name(visibility_level)} group" do
+          import_request
+
+          group = parent_group.children.find_by(name: 'test-group-import')
+          expect(group.visibility_level).to eq visibility_level
+        end
+      end
+
+      [
+        Gitlab::VisibilityLevel::PUBLIC,
+        Gitlab::VisibilityLevel::INTERNAL,
+        Gitlab::VisibilityLevel::PRIVATE
+      ].each do |visibility_level|
+        context "when the parent is #{Gitlab::VisibilityLevel.level_name(visibility_level)}" do
+          include_examples 'is created with the parent visibility level', visibility_level
+        end
+      end
+    end
+
+    context 'when supplied invalid params' do
+      subject(:import_request) do
+        upload_archive(
+          file_upload,
+          workhorse_headers,
+          { path: '', name: '' }
+        )
+      end
+
+      it 'responds with an error' do
+        expect { import_request }.not_to change { Group.count }
+
+        expect(flash[:alert])
+          .to include('Group could not be imported', "Name can't be blank", 'Group URL is too short')
+      end
+    end
+
+    context 'when the user is not authorized to create groups' do
+      let(:request_params) { { path: 'test-group-import', name: 'test-group-import' } }
+      let(:user) { create(:user, can_create_group: false) }
+
+      it 'returns an error' do
+        expect { import_request }.not_to change { Group.count }
+
+        expect(flash[:alert]).to eq 'Group could not be imported: You don’t have permission to create groups.'
+      end
+    end
+
+    context 'when the requests exceed the rate limit' do
+      let(:request_params) { { path: 'test-group-import', name: 'test-group-import' } }
+
+      before do
+        allow(Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(true)
+      end
+
+      it 'throttles the requests' do
+        import_request
+
+        expect(response).to have_gitlab_http_status(:found)
+        expect(flash[:alert]).to eq 'This endpoint has been requested too many times. Try again later.'
+      end
+    end
+
+    context 'when group import FF is disabled' do
+      let(:request_params) { { path: 'test-group-import', name: 'test-group-import' } }
+
+      before do
+        stub_feature_flags(group_import_export: false)
+      end
+
+      it 'returns an error' do
+        expect { import_request }.not_to change { Group.count }
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when the parent group is invalid' do
+      let(:request_params) { { path: 'test-group-import', name: 'test-group-import', parent_id: -1 } }
+
+      it 'does not create a new group' do
+        expect { import_request }.not_to change { Group.count }
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when the user is not an owner of the parent group' do
+      let(:request_params) { { path: 'test-group-import', name: 'test-group-import', parent_id: parent_group.id } }
+      let(:parent_group) { create(:group) }
+
+      it 'returns an error' do
+        expect { import_request }.not_to change { parent_group.children.reload.count }
+
+        expect(flash[:alert]).to include "You don’t have permission to create a subgroup in this group"
+      end
+    end
+  end
+
+  describe 'POST authorize' do
+    let_it_be(:user) { create(:user) }
+
+    before do
+      login_as(user)
+    end
+
+    context 'when using a workhorse header' do
+      subject(:authorize_request) { post authorize_import_gitlab_group_path, headers: workhorse_headers }
+
+      it 'authorizes the request' do
+        authorize_request
+
+        expect(response).to have_gitlab_http_status :ok
+        expect(response.media_type).to eq Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+        expect(json_response['TempPath']).to eq ImportExportUploader.workhorse_local_upload_path
+      end
+    end
+
+    context 'when the request bypasses gitlab-workhorse' do
+      subject(:authorize_request) { post authorize_import_gitlab_group_path }
+
+      it 'rejects the request' do
+        expect { authorize_request }.to raise_error(JWT::DecodeError)
+      end
+    end
+
+    context 'when direct upload is enabled' do
+      subject(:authorize_request) { post authorize_import_gitlab_group_path, headers: workhorse_headers }
+
+      before do
+        stub_uploads_object_storage(ImportExportUploader, enabled: true, direct_upload: true)
+      end
+
+      it 'accepts the request and stores the files' do
+        authorize_request
+
+        expect(response).to have_gitlab_http_status :ok
+        expect(response.media_type).to eq Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+        expect(json_response).not_to have_key 'TempPath'
+
+        expect(json_response['RemoteObject'].keys)
+          .to include('ID', 'GetURL', 'StoreURL', 'DeleteURL', 'MultipartUpload')
+      end
+    end
+
+    context 'when direct upload is disabled' do
+      subject(:authorize_request) { post authorize_import_gitlab_group_path, headers: workhorse_headers }
+
+      before do
+        stub_uploads_object_storage(ImportExportUploader, enabled: true, direct_upload: false)
+      end
+
+      it 'handles the local file' do
+        authorize_request
+
+        expect(response).to have_gitlab_http_status :ok
+        expect(response.media_type).to eq Gitlab::Workhorse::INTERNAL_API_CONTENT_TYPE
+
+        expect(json_response['TempPath']).to eq ImportExportUploader.workhorse_local_upload_path
+        expect(json_response['RemoteObject']).to be_nil
+      end
+    end
+  end
+end

spec/services/groups/import_export/import_service_spec.rb

@@ -3,6 +3,37 @@
 require 'spec_helper'
 
 describe Groups::ImportExport::ImportService do
+  describe '#async_execute' do
+    let_it_be(:user) { create(:user) }
+    let_it_be(:group) { create(:group) }
+
+    context 'when the job can be successfully scheduled' do
+      subject(:import_service) { described_class.new(group: group, user: user) }
+
+      it 'enqueues an import job' do
+        expect(GroupImportWorker).to receive(:perform_async).with(user.id, group.id)
+
+        import_service.async_execute
+      end
+
+      it 'returns truthy' do
+        expect(import_service.async_execute).to be_truthy
+      end
+    end
+
+    context 'when the job cannot be scheduled' do
+      subject(:import_service) { described_class.new(group: group, user: user) }
+
+      before do
+        allow(GroupImportWorker).to receive(:perform_async).and_return(nil)
+      end
+
+      it 'returns falsey' do
+        expect(import_service.async_execute).to be_falsey
+      end
+    end
+  end
+
   context 'with group_import_ndjson feature flag disabled' do
     let(:user) { create(:admin) }
     let(:group) { create(:group) }