Commit efb71c70 authored by George Koltsov's avatar George Koltsov

Add Group relations export API

  - Add a set of Group endpoints to initiate relations export
  - Add a set of Sidekiq workers to perform relations export
  - Upload exported ndjson.gz to ObjectStorage

Changelog: added
parent 2c4abf28
......@@ -36,6 +36,15 @@ module BulkImports
end
end
def self.config(exportable)
case exportable
when ::Project
Exports::ProjectConfig.new(exportable)
when ::Group
Exports::GroupConfig.new(exportable)
end
end
def exportable_relation?
return unless exportable
......@@ -54,12 +63,7 @@ module BulkImports
def config
strong_memoize(:config) do
case exportable
when ::Project
Exports::ProjectConfig.new(exportable)
when ::Group
Exports::GroupConfig.new(exportable)
end
self.class.config(exportable)
end
end
end
......
......@@ -13,11 +13,6 @@ module BulkImports
attributes_finder.find_root(exportable_class_sym)
end
def validate_user_permissions!(user)
user.can?(ability, exportable) ||
raise(::Gitlab::ImportExport::Error.permission_error(user, exportable))
end
def export_path
strong_memoize(:export_path) do
relative_path = File.join(base_export_path, SecureRandom.hex)
......@@ -56,10 +51,6 @@ module BulkImports
raise NotImplementedError
end
def ability
raise NotImplementedError
end
def base_export_path
raise NotImplementedError
end
......
......@@ -3,8 +3,6 @@
module BulkImports
module Exports
class GroupConfig < BaseConfig
private
def base_export_path
exportable.full_path
end
......@@ -12,10 +10,6 @@ module BulkImports
def import_export_yaml
::Gitlab::ImportExport.group_config_file
end
def ability
:admin_group
end
end
end
end
......@@ -3,8 +3,6 @@
module BulkImports
module Exports
class ProjectConfig < BaseConfig
private
def base_export_path
exportable.disk_path
end
......@@ -12,10 +10,6 @@ module BulkImports
def import_export_yaml
::Gitlab::ImportExport.config_file
end
def ability
:admin_project
end
end
end
end
......@@ -704,6 +704,10 @@ class Group < Namespace
Gitlab::ServiceDesk.supported? && all_projects.service_desk_enabled.exists?
end
def to_ability_name
model_name.singular
end
private
def update_two_factor_requirement
......
# frozen_string_literal: true
module BulkImports
class ExportService
def initialize(exportable:, user:)
@exportable = exportable
@current_user = user
end
def execute
Export.config(exportable).exportable_relations.each do |relation|
RelationExportWorker.perform_async(current_user.id, exportable.id, exportable.class.name, relation)
end
ServiceResponse.success
rescue StandardError => e
ServiceResponse.error(
message: e.class,
http_status: :unprocessable_entity
)
end
private
attr_reader :exportable, :current_user
end
end
# frozen_string_literal: true
module BulkImports
class RelationExportService
include Gitlab::ImportExport::CommandLineUtil
def initialize(user, exportable, relation, jid)
@user = user
@exportable = exportable
@relation = relation
@jid = jid
end
def execute
find_or_create_export! do |export|
remove_existing_export_file!(export)
serialize_relation_to_file(export.relation_definition)
compress_exported_relation
upload_compressed_file(export)
end
end
private
attr_reader :user, :exportable, :relation, :jid
def find_or_create_export!
validate_user_permissions!
export = exportable.bulk_import_exports.safe_find_or_create_by!(relation: relation)
export.update!(status_event: 'start', jid: jid)
yield export
export.update!(status_event: 'finish', error: nil)
rescue StandardError => e
Gitlab::ErrorTracking.track_exception(e, exportable_id: exportable.id, exportable_type: exportable.class.name)
export&.update(status_event: 'fail_op', error: e.class)
end
def validate_user_permissions!
ability = "admin_#{exportable.to_ability_name}"
user.can?(ability, exportable) ||
raise(::Gitlab::ImportExport::Error.permission_error(user, exportable))
end
def remove_existing_export_file!(export)
upload = export.upload
return unless upload&.export_file&.file
upload.remove_export_file!
upload.save!
end
def serialize_relation_to_file(relation_definition)
serializer.serialize_relation(relation_definition)
end
def compress_exported_relation
gzip(dir: export_path, filename: ndjson_filename)
end
def upload_compressed_file(export)
compressed_filename = File.join(export_path, "#{ndjson_filename}.gz")
upload = ExportUpload.find_or_initialize_by(export_id: export.id) # rubocop: disable CodeReuse/ActiveRecord
File.open(compressed_filename) { |file| upload.export_file = file }
upload.save!
end
def export_config
@export_config ||= Export.config(exportable)
end
def export_path
@export_path ||= export_config.export_path
end
def exportable_tree
@exportable_tree ||= export_config.exportable_tree
end
# rubocop: disable CodeReuse/Serializer
def serializer
@serializer ||= ::Gitlab::ImportExport::JSON::StreamingSerializer.new(
exportable,
exportable_tree,
json_writer,
exportable_path: ''
)
end
# rubocop: enable CodeReuse/Serializer
def json_writer
@json_writer ||= ::Gitlab::ImportExport::JSON::NdjsonWriter.new(export_path)
end
def ndjson_filename
@ndjson_filename ||= "#{relation}.ndjson"
end
end
end
......@@ -1837,6 +1837,16 @@
:idempotent:
:tags:
- :exclude_from_kubernetes
- :name: bulk_imports_relation_export
:worker_name: BulkImports::RelationExportWorker
:feature_category: :importers
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
:tags:
- :exclude_from_kubernetes
- :name: chat_notification
:worker_name: ChatNotificationWorker
:feature_category: :chatops
......
# frozen_string_literal: true
module BulkImports
class RelationExportWorker
include ApplicationWorker
include ExceptionBacktrace
idempotent!
loggable_arguments 2, 3
feature_category :importers
tags :exclude_from_kubernetes
sidekiq_options status_expiration: StuckExportJobsWorker::EXPORT_JOBS_EXPIRATION
def perform(user_id, exportable_id, exportable_class, relation)
user = User.find(user_id)
exportable = exportable(exportable_id, exportable_class)
RelationExportService.new(user, exportable, relation, jid).execute
end
private
def exportable(exportable_id, exportable_class)
exportable_class.classify.constantize.find(exportable_id)
end
end
end
---
title: Add Group relations export API
merge_request: 59978
author:
type: added
......@@ -58,6 +58,8 @@
- 1
- - bulk_imports_pipeline
- 1
- - bulk_imports_relation_export
- 1
- - chaos
- 2
- - chat_notification
......
# frozen_string_literal: true
module API
module Entities
module BulkImports
class ExportStatus < Grape::Entity
expose :relation
expose :status
expose :error
expose :updated_at
end
end
end
end
......@@ -43,6 +43,43 @@ module API
render_api_error!(message: 'Group export could not be started.')
end
end
desc 'Start relations export' do
detail 'This feature was introduced in GitLab 13.12'
end
post ':id/export_relations' do
response = ::BulkImports::ExportService.new(exportable: user_group, user: current_user).execute
if response.success?
accepted!
else
render_api_error!(message: 'Group relations export could not be started.')
end
end
desc 'Download relations export' do
detail 'This feature was introduced in GitLab 13.12'
end
params do
requires :relation, type: String, desc: 'Group relation name'
end
get ':id/export_relations/download' do
export = user_group.bulk_import_exports.find_by_relation(params[:relation])
file = export&.upload&.export_file
if file
present_carrierwave_file!(file)
else
render_api_error!('404 Not found', 404)
end
end
desc 'Relations export status' do
detail 'This feature was introduced in GitLab 13.12'
end
get ':id/export_relations/status' do
present user_group.bulk_import_exports, with: Entities::BulkImports::ExportStatus
end
end
end
end
......@@ -14,6 +14,19 @@ module Gitlab
untar_with_options(archive: archive, dir: dir, options: 'zxf')
end
def gzip(dir:, filename:)
filepath = File.join(dir, filename)
cmd = %W(gzip #{filepath})
_, status = Gitlab::Popen.popen(cmd)
if status == 0
status
else
raise Gitlab::ImportExport::Error.file_compression_error
end
end
def mkdir_p(path)
FileUtils.mkdir_p(path, mode: DEFAULT_DIR_MODE)
FileUtils.chmod(DEFAULT_DIR_MODE, path)
......
......@@ -38,16 +38,6 @@ module Gitlab
end
end
private
attr_reader :json_writer, :relations_schema, :exportable
def serialize_root
attributes = exportable.as_json(
relations_schema.merge(include: nil, preloads: nil))
json_writer.write_attributes(@exportable_path, attributes)
end
def serialize_relation(definition)
raise ArgumentError, 'definition needs to be Hash' unless definition.is_a?(Hash)
raise ArgumentError, 'definition needs to have exactly one Hash element' unless definition.one?
......@@ -64,6 +54,16 @@ module Gitlab
end
end
private
attr_reader :json_writer, :relations_schema, :exportable
def serialize_root
attributes = exportable.as_json(
relations_schema.merge(include: nil, preloads: nil))
json_writer.write_attributes(@exportable_path, attributes)
end
def serialize_many_relations(key, records, options)
enumerator = Enumerator.new do |items|
key_preloads = preloads&.dig(key)
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe API::Entities::BulkImports::ExportStatus do
let_it_be(:export) { create(:bulk_import_export) }
let(:entity) { described_class.new(export, request: double) }
subject { entity.as_json }
it 'has the correct attributes' do
expect(subject).to eq({
relation: export.relation,
status: export.status,
error: export.error,
updated_at: export.updated_at
})
end
end
......@@ -35,4 +35,19 @@ RSpec.describe Gitlab::ImportExport::CommandLineUtil do
it 'has the right mask for uploads' do
expect(file_permissions("#{path}/uploads")).to eq(0755) # originally 555
end
describe '#gzip' do
it 'compresses specified file' do
tempfile = Tempfile.new('test', path)
filename = File.basename(tempfile.path)
subject.gzip(dir: path, filename: filename)
end
context 'when exception occurs' do
it 'raises an exception' do
expect { subject.gzip(dir: path, filename: 'test') }.to raise_error(Gitlab::ImportExport::Error)
end
end
end
end
......@@ -30,24 +30,6 @@ RSpec.describe BulkImports::Exports::GroupConfig do
end
end
describe '#validate_user_permissions' do
let_it_be(:user) { create(:user) }
context 'when user cannot admin project' do
it 'returns false' do
expect { subject.validate_user_permissions!(user) }.to raise_error(Gitlab::ImportExport::Error)
end
end
context 'when user can admin project' do
it 'returns true' do
exportable.add_owner(user)
expect(subject.validate_user_permissions!(user)).to eq(true)
end
end
end
describe '#exportable_relations' do
it 'returns a list of top level exportable relations' do
expect(subject.exportable_relations).to include('milestones', 'badges', 'boards', 'labels')
......
......@@ -30,24 +30,6 @@ RSpec.describe BulkImports::Exports::ProjectConfig do
end
end
describe '#validate_user_permissions' do
let_it_be(:user) { create(:user) }
context 'when user cannot admin project' do
it 'returns false' do
expect { subject.validate_user_permissions!(user) }.to raise_error(Gitlab::ImportExport::Error)
end
end
context 'when user can admin project' do
it 'returns true' do
exportable.add_maintainer(user)
expect(subject.validate_user_permissions!(user)).to eq(true)
end
end
end
describe '#exportable_relations' do
it 'returns a list of top level exportable relations' do
expect(subject.exportable_relations).to include('issues', 'labels', 'milestones', 'merge_requests')
......
......@@ -2390,4 +2390,12 @@ RSpec.describe Group do
it { is_expected.to eq(Set.new([child_1.id])) }
end
describe '#to_ability_name' do
it 'returns group' do
group = build(:group)
expect(group.to_ability_name).to eq('group')
end
end
end
......@@ -178,4 +178,74 @@ RSpec.describe API::GroupExport do
end
end
end
describe 'relations export' do
let(:path) { "/groups/#{group.id}/export_relations" }
let(:download_path) { "/groups/#{group.id}/export_relations/download?relation=labels" }
let(:status_path) { "/groups/#{group.id}/export_relations/status" }
before do
stub_feature_flags(group_import_export: true)
group.add_owner(user)
end
describe 'POST /groups/:id/export_relations' do
it 'accepts the request' do
post api(path, user)
expect(response).to have_gitlab_http_status(:accepted)
end
context 'when response is not success' do
it 'returns api error' do
allow_next_instance_of(BulkImports::ExportService) do |service|
allow(service).to receive(:execute).and_return(ServiceResponse.error(message: 'error', http_status: :error))
end
post api(path, user)
expect(response).to have_gitlab_http_status(:error)
end
end
end
describe 'GET /groups/:id/export_relations/download' do
let(:export) { create(:bulk_import_export, group: group, relation: 'labels') }
let(:upload) { create(:bulk_import_export_upload, export: export) }
context 'when export file exists' do
it 'downloads exported group archive' do
upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/labels.ndjson.gz'))
get api(download_path, user)
expect(response).to have_gitlab_http_status(:ok)
end
end
context 'when export_file.file does not exist' do
it 'returns 404' do
allow(upload).to receive(:export_file).and_return(nil)
get api(download_path, user)
expect(response).to have_gitlab_http_status(:not_found)
end
end
end
describe 'GET /groups/:id/export_relations/status' do
it 'returns a list of relation export statuses' do
create(:bulk_import_export, :started, group: group, relation: 'labels')
create(:bulk_import_export, :finished, group: group, relation: 'milestones')
create(:bulk_import_export, :failed, group: group, relation: 'badges')
get api(status_path, user)
expect(response).to have_gitlab_http_status(:ok)
expect(json_response.pluck('relation')).to contain_exactly('labels', 'milestones', 'badges')
expect(json_response.pluck('status')).to contain_exactly(-1, 0, 1)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::ExportService do
let_it_be(:group) { create(:group) }
let_it_be(:user) { create(:user) }
before do
group.add_owner(user)
end
subject { described_class.new(exportable: group, user: user) }
describe '#execute' do
it 'schedules RelationExportWorker for each top level relation' do
expect(subject).to receive(:execute).and_return(ServiceResponse.success).and_call_original
top_level_relations = BulkImports::Export.config(group).exportable_relations
top_level_relations.each do |relation|
expect(BulkImports::RelationExportWorker)
.to receive(:perform_async)
.with(user.id, group.id, group.class.name, relation)
end
subject.execute
end
context 'when exception occurs' do
it 'does not schedule RelationExportWorker' do
service = described_class.new(exportable: nil, user: user)
expect(service)
.to receive(:execute)
.and_return(ServiceResponse.error(message: 'Gitlab::ImportExport::Error', http_status: :unprocessible_entity))
.and_call_original
expect(BulkImports::RelationExportWorker).not_to receive(:perform_async)
service.execute
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::RelationExportService do
let_it_be(:jid) { 'jid' }
let_it_be(:relation) { 'labels' }
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:label) { create(:group_label, group: group) }
let_it_be(:export_path) { "#{Dir.tmpdir}/relation_export_service_spec/tree" }
let_it_be_with_reload(:export) { create(:bulk_import_export, group: group, relation: relation) }
before do
group.add_owner(user)
allow(export).to receive(:export_path).and_return(export_path)
end
after :all do
FileUtils.rm_rf(export_path)
end
subject { described_class.new(user, group, relation, jid) }
describe '#execute' do
it 'exports specified relation and marks export as finished' do
subject.execute
expect(export.reload.upload.export_file).to be_present
expect(export.finished?).to eq(true)
end
it 'removes temp export files' do
subject.execute
expect(Dir.exist?(export_path)).to eq(false)
end
it 'exports specified relation and marks export as finished' do
subject.execute
expect(export.upload.export_file).to be_present
end
context 'when export record does not exist' do
let(:another_group) { create(:group) }
subject { described_class.new(user, another_group, relation, jid) }
it 'creates export record' do
another_group.add_owner(user)
expect { subject.execute }
.to change { another_group.bulk_import_exports.count }
.from(0)
.to(1)
end
end
context 'when there is existing export present' do
let(:upload) { create(:bulk_import_export_upload, export: export) }
it 'removes existing export before exporting' do
upload.update!(export_file: fixture_file_upload('spec/fixtures/bulk_imports/labels.ndjson.gz'))
expect_any_instance_of(BulkImports::ExportUpload) do |upload|
expect(upload).to receive(:remove_export_file!)
end
subject.execute
end
end
context 'when exception occurs during export' do
shared_examples 'tracks exception' do |exception_class|
it 'tracks exception' do
expect(Gitlab::ErrorTracking)
.to receive(:track_exception)
.with(exception_class, exportable_id: group.id, exportable_type: group.class.name)
.and_call_original
subject.execute
end
end
before do
allow_next_instance_of(BulkImports::ExportUpload) do |upload|
allow(upload).to receive(:save!).and_raise(StandardError)
end
end
it 'marks export as failed' do
subject.execute
expect(export.reload.failed?).to eq(true)
end
include_examples 'tracks exception', StandardError
context 'when passed relation is not supported' do
let(:relation) { 'unsupported' }
include_examples 'tracks exception', ActiveRecord::RecordInvalid
end
context 'when user is not allowed to perform export' do
let(:another_user) { create(:user) }
subject { described_class.new(another_user, group, relation, jid) }
include_examples 'tracks exception', Gitlab::ImportExport::Error
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::RelationExportWorker do
let_it_be(:jid) { 'jid' }
let_it_be(:relation) { 'labels' }
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group) }
let(:job_args) { [user.id, group.id, group.class.name, relation] }
describe '#perform' do
include_examples 'an idempotent worker' do
context 'when export record does not exist' do
let(:another_group) { create(:group) }
let(:job_args) { [user.id, another_group.id, another_group.class.name, relation] }
it 'creates export record' do
another_group.add_owner(user)
expect { perform_multiple(job_args) }
.to change { another_group.bulk_import_exports.count }
.from(0)
.to(1)
end
end
it 'executes RelationExportService' do
group.add_owner(user)
service = instance_double(BulkImports::RelationExportService)
expect(BulkImports::RelationExportService)
.to receive(:new)
.with(user, group, relation, anything)
.twice
.and_return(service)
expect(service)
.to receive(:execute)
.twice
perform_multiple(job_args)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment