Merge branch 'jh-group_export_ui_backend' into 'master'

Add endpoints for the group export UI

See merge request gitlab-org/gitlab!28568

app/controllers/groups_controller.rb

@@ -6,18 +6,20 @@ class GroupsController < Groups::ApplicationController
   include ParamsBackwardCompatibility
   include PreviewMarkdown
   include RecordUserLastActivity
+  include SendFileUpload
   extend ::Gitlab::Utils::Override
 
   respond_to :html
 
   prepend_before_action(only: [:show, :issues]) { authenticate_sessionless_user!(:rss) }
   prepend_before_action(only: [:issues_calendar]) { authenticate_sessionless_user!(:ics) }
+  prepend_before_action :ensure_export_enabled, only: [:export, :download_export]
 
   before_action :authenticate_user!, only: [:new, :create]
   before_action :group, except: [:index, :new, :create]
 
   # Authorize
-  before_action :authorize_admin_group!, only: [:edit, :update, :destroy, :projects, :transfer]
+  before_action :authorize_admin_group!, only: [:edit, :update, :destroy, :projects, :transfer, :export, :download_export]
   before_action :authorize_create_group!, only: [:new]
 
   before_action :group_projects, only: [:projects, :activity, :issues, :merge_requests]
@@ -29,6 +31,8 @@ class GroupsController < Groups::ApplicationController
     push_frontend_feature_flag(:vue_issuables_list, @group)
   end
 
+  before_action :export_rate_limit, only: [:export, :download_export]
+
   skip_cross_project_access_check :index, :new, :create, :edit, :update,
                                   :destroy, :projects
   # When loading show as an atom feed, we render events that could leak cross
@@ -134,6 +138,25 @@ class GroupsController < Groups::ApplicationController
   end
   # rubocop: enable CodeReuse/ActiveRecord
 
+  def export
+    export_service = Groups::ImportExport::ExportService.new(group: @group, user: current_user)
+
+    if export_service.async_execute
+      redirect_to edit_group_path(@group), notice: _('Group export started.')
+    else
+      redirect_to edit_group_path(@group), alert: _('Group export could not be started.')
+    end
+  end
+
+  def download_export
+    if @group.export_file_exists?
+      send_upload(@group.export_file, attachment: @group.export_file.filename)
+    else
+      redirect_to edit_group_path(@group),
+        alert: _('Group export link has expired. Please generate a new export from your group settings.')
+    end
+  end
+
   protected
 
   def render_show_html
@@ -234,6 +257,21 @@ class GroupsController < Groups::ApplicationController
     url_for(safe_params)
   end
 
+  def export_rate_limit
+    prefixed_action = "group_#{params[:action]}".to_sym
+
+    if Gitlab::ApplicationRateLimiter.throttled?(prefixed_action, scope: [current_user, prefixed_action, @group])
+      Gitlab::ApplicationRateLimiter.log_request(request, "#{prefixed_action}_request_limit".to_sym, current_user)
+
+      flash[:alert] = _('This endpoint has been requested too many times. Try again later.')
+      redirect_to edit_group_path(@group)
+    end
+  end
+
+  def ensure_export_enabled
+    render_404 unless Feature.enabled?(:group_import_export, @group, default_enabled: true)
+  end
+
   private
 
   def groups

app/services/groups/import_export/export_service.rb

@@ -10,9 +10,15 @@ module Groups
         @shared       = @params[:shared] || Gitlab::ImportExport::Shared.new(@group)
       end
 
+      def async_execute
+        GroupExportWorker.perform_async(@current_user.id, @group.id, @params)
+      end
+
       def execute
         validate_user_permissions
 
+        remove_existing_export! if @group.export_file_exists?
+
         save!
       ensure
         cleanup
@@ -30,6 +36,13 @@ module Groups
         end
       end
 
+      def remove_existing_export!
+        import_export_upload = @group.import_export_upload
+
+        import_export_upload.remove_export_file!
+        import_export_upload.save
+      end
+
       def save!
         if savers.all?(&:save)
           notify_success

config/routes/group.rb

@@ -13,6 +13,9 @@ constraints(::Constraints::GroupUrlConstrainer.new) do
       get :details, as: :details_group
       get :activity, as: :activity_group
       put :transfer, as: :transfer_group
+      post :export, as: :export_group
+      get :download_export, as: :download_export_group
+
       # TODO: Remove as part of refactor in https://gitlab.com/gitlab-org/gitlab-foss/issues/49693
       get 'shared', action: :show, as: :group_shared
       get 'archived', action: :show, as: :group_archived

lib/api/group_export.rb

@@ -27,9 +27,13 @@ module API
         detail 'This feature was introduced in GitLab 12.5.'
       end
       post ':id/export' do
-        GroupExportWorker.perform_async(current_user.id, user_group.id, params) # rubocop:disable CodeReuse/Worker
+        export_service = ::Groups::ImportExport::ExportService.new(group: user_group, user: current_user)
 
-        accepted!
+        if export_service.async_execute
+          accepted!
+        else
+          render_api_error!(message: 'Group export could not be started.')
+        end
       end
     end
   end

lib/gitlab/application_rate_limiter.rb

@@ -25,7 +25,9 @@ module Gitlab
           project_generate_new_export: { threshold: 1, interval: 5.minutes },
           project_import:              { threshold: 30, interval: 5.minutes },
           play_pipeline_schedule:      { threshold: 1, interval: 1.minute },
-          show_raw_controller:         { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute }
+          show_raw_controller:         { threshold: -> { Gitlab::CurrentSettings.current_application_settings.raw_blob_request_limit }, interval: 1.minute },
+          group_export:                { threshold: 1, interval: 5.minutes },
+          group_download_export:       { threshold: 10, interval: 10.minutes }
         }.freeze
       end
 

locale/gitlab.pot

@@ -10112,6 +10112,15 @@ msgstr ""
 msgid "Group details"
 msgstr ""
 
+msgid "Group export could not be started."
+msgstr ""
+
+msgid "Group export link has expired. Please generate a new export from your group settings."
+msgstr ""
+
+msgid "Group export started."
+msgstr ""
+
 msgid "Group has been already marked for deletion"
 msgstr ""
 

spec/controllers/groups_controller_spec.rb

@@ -764,6 +764,136 @@ describe GroupsController do
     end
   end
 
+  describe 'POST #export' do
+    context 'when the group export feature flag is not enabled' do
+      before do
+        sign_in(admin)
+        stub_feature_flags(group_import_export: false)
+      end
+
+      it 'returns a not found error' do
+        post :export, params: { id: group.to_param }
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when the user does not have permission to export the group' do
+      before do
+        sign_in(guest)
+      end
+
+      it 'returns an error' do
+        post :export, params: { id: group.to_param }
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when supplied valid params' do
+      before do
+        sign_in(admin)
+      end
+
+      it 'triggers the export job' do
+        expect(GroupExportWorker).to receive(:perform_async).with(admin.id, group.id, {})
+
+        post :export, params: { id: group.to_param }
+      end
+
+      it 'redirects to the edit page' do
+        post :export, params: { id: group.to_param }
+
+        expect(response).to have_gitlab_http_status(:found)
+      end
+    end
+
+    context 'when the endpoint receives requests above the rate limit' do
+      before do
+        sign_in(admin)
+        allow(Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(true)
+      end
+
+      it 'throttles the endpoint' do
+        post :export, params: { id: group.to_param }
+
+        expect(flash[:alert]).to eq('This endpoint has been requested too many times. Try again later.')
+        expect(response).to have_gitlab_http_status(:found)
+      end
+    end
+  end
+
+  describe 'GET #download_export' do
+    context 'when there is a file available to download' do
+      let(:export_file) { fixture_file_upload('spec/fixtures/group_export.tar.gz') }
+
+      before do
+        sign_in(admin)
+        create(:import_export_upload, group: group, export_file: export_file)
+      end
+
+      it 'sends the file' do
+        get :download_export, params: { id: group.to_param }
+
+        expect(response.body).to eq export_file.tempfile.read
+      end
+    end
+
+    context 'when there is no file available to download' do
+      before do
+        sign_in(admin)
+      end
+
+      it 'returns not found' do
+        get :download_export, params: { id: group.to_param }
+
+        expect(flash[:alert])
+          .to eq 'Group export link has expired. Please generate a new export from your group settings.'
+
+        expect(response).to redirect_to(edit_group_path(group))
+      end
+    end
+
+    context 'when the group export feature flag is not enabled' do
+      before do
+        sign_in(admin)
+        stub_feature_flags(group_import_export: false)
+      end
+
+      it 'returns a not found error' do
+        post :export, params: { id: group.to_param }
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when the user does not have the required permissions' do
+      before do
+        sign_in(guest)
+      end
+
+      it 'returns not_found' do
+        get :download_export, params: { id: group.to_param }
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'when the endpoint receives requests above the rate limit' do
+      before do
+        sign_in(admin)
+        allow(Gitlab::ApplicationRateLimiter).to receive(:throttled?).and_return(true)
+      end
+
+      it 'throttles the endpoint' do
+        get :download_export, params: { id: group.to_param }
+
+        expect(flash[:alert]).to eq('This endpoint has been requested too many times. Try again later.')
+        expect(response).to have_gitlab_http_status(:found)
+      end
+    end
+  end
+
   context 'token authentication' do
     it_behaves_like 'authenticates sessionless user', :show, :atom, public: true do
       before do

spec/requests/api/group_export_spec.rb

@@ -102,6 +102,19 @@ describe API::GroupExport do
         end
       end
 
+      context 'when the export cannot be started' do
+        before do
+          group.add_owner(user)
+          allow(GroupExportWorker).to receive(:perform_async).and_return(nil)
+        end
+
+        it 'returns an error' do
+          post api(path, user)
+
+          expect(response).to have_gitlab_http_status(:error)
+        end
+      end
+
       context 'when user is not a group owner' do
         before do
           group.add_developer(user)

spec/services/groups/import_export/export_service_spec.rb

@@ -3,6 +3,37 @@
 require 'spec_helper'
 
 describe Groups::ImportExport::ExportService do
+  describe '#async_execute' do
+    let(:user) { create(:user) }
+    let(:group) { create(:group) }
+
+    context 'when the job can be successfully scheduled' do
+      let(:export_service) { described_class.new(group: group, user: user) }
+
+      it 'enqueues an export job' do
+        expect(GroupExportWorker).to receive(:perform_async).with(user.id, group.id, {})
+
+        export_service.async_execute
+      end
+
+      it 'returns truthy' do
+        expect(export_service.async_execute).to be_present
+      end
+    end
+
+    context 'when the job cannot be scheduled' do
+      let(:export_service) { described_class.new(group: group, user: user) }
+
+      before do
+        allow(GroupExportWorker).to receive(:perform_async).and_return(nil)
+      end
+
+      it 'returns falsey' do
+        expect(export_service.async_execute).to be_falsey
+      end
+    end
+  end
+
   describe '#execute' do
     let!(:user) { create(:user) }
     let(:group) { create(:group) }
@@ -103,5 +134,23 @@ describe Groups::ImportExport::ExportService do
         end
       end
     end
+
+    context 'when there is an existing export file' do
+      subject(:export_service) { described_class.new(group: group, user: user) }
+
+      let(:import_export_upload) do
+        create(
+          :import_export_upload,
+          group: group,
+          export_file: fixture_file_upload('spec/fixtures/group_export.tar.gz')
+        )
+      end
+
+      it 'removes it' do
+        existing_file = import_export_upload.export_file
+
+        expect { export_service.execute }.to change { existing_file.file }.to(be_nil)
+      end
+    end
   end
 end