Commit 50099f46 authored by James Lopez's avatar James Lopez

Merge branch 'georgekoltsov/group-mig-etl-inline-methods' into 'master'

Update BulkImports::Pipeline to make use of DSL methods optional

See merge request gitlab-org/gitlab!54368
parents b327db74 45bc4ca5
......@@ -7,10 +7,6 @@ module EE
class EpicAwardEmojiLoader
NotAllowedError = Class.new(StandardError)
def initialize(options = {})
@options = options
end
# rubocop: disable CodeReuse/ActiveRecord
def load(context, data)
return unless data
......
......@@ -7,10 +7,6 @@ module EE
class EpicsLoader
NotAllowedError = Class.new(StandardError)
def initialize(options = {})
@options = options
end
def load(context, data)
raise NotAllowedError unless authorized?(context)
......
......@@ -5,8 +5,6 @@ module EE
module Groups
module Transformers
class EpicAttributesTransformer
def initialize(*args); end
def transform(context, data)
data
.then { |data| add_group_id(context, data) }
......
# frozen_string_literal: true
module BulkImports
module Common
module Loaders
class EntityLoader
def initialize(*args); end
def load(context, entity)
context.bulk_import.entities.create!(entity)
end
end
end
end
end
......@@ -4,8 +4,6 @@ module BulkImports
module Common
module Transformers
class AwardEmojiTransformer
def initialize(*args); end
def transform(context, data)
user = find_user(context, data&.dig('user', 'public_email')) || context.current_user
......
......@@ -14,10 +14,6 @@ module BulkImports
/\Aremote_\w+_(url|urls|request_header)\Z/ # carrierwave automatically creates these attribute methods for uploads
).freeze
def initialize(options = {})
@options = options
end
def transform(context, data)
data.each_with_object({}) do |(key, value), result|
prohibited = prohibited_key?(key)
......
......@@ -4,8 +4,6 @@ module BulkImports
module Groups
module Extractors
class SubgroupsExtractor
def initialize(*args); end
def extract(context)
encoded_parent_path = ERB::Util.url_encode(context.entity.source_full_path)
......
......@@ -4,10 +4,6 @@ module BulkImports
module Groups
module Loaders
class GroupLoader
def initialize(options = {})
@options = options
end
def load(context, data)
return unless user_can_create_group?(context.current_user, data)
......
# frozen_string_literal: true
module BulkImports
module Groups
module Loaders
class LabelsLoader
def initialize(*); end
def load(context, data)
Labels::CreateService.new(data).execute(group: context.group)
end
end
end
end
end
# frozen_string_literal: true
module BulkImports
module Groups
module Loaders
class MembersLoader
def initialize(*); end
def load(context, data)
return unless data
context.group.members.create!(data)
end
end
end
end
end
......@@ -11,7 +11,9 @@ module BulkImports
transformer Common::Transformers::ProhibitedAttributesTransformer
loader BulkImports::Groups::Loaders::LabelsLoader
def load(context, data)
Labels::CreateService.new(data).execute(group: context.group)
end
def after_run(extracted_data)
context.entity.update_tracker_for(
......
......@@ -12,7 +12,11 @@ module BulkImports
transformer Common::Transformers::ProhibitedAttributesTransformer
transformer BulkImports::Groups::Transformers::MemberAttributesTransformer
loader BulkImports::Groups::Loaders::MembersLoader
def load(context, data)
return unless data
context.group.members.create!(data)
end
def after_run(extracted_data)
context.entity.update_tracker_for(
......
......@@ -9,7 +9,10 @@ module BulkImports
extractor BulkImports::Groups::Extractors::SubgroupsExtractor
transformer Common::Transformers::ProhibitedAttributesTransformer
transformer BulkImports::Groups::Transformers::SubgroupToEntityTransformer
loader BulkImports::Common::Loaders::EntityLoader
def load(context, data)
context.bulk_import.entities.create!(data)
end
end
end
end
......
......@@ -4,10 +4,6 @@ module BulkImports
module Groups
module Transformers
class GroupAttributesTransformer
def initialize(options = {})
@options = options
end
def transform(context, data)
import_entity = context.entity
......
......@@ -4,8 +4,6 @@ module BulkImports
module Groups
module Transformers
class MemberAttributesTransformer
def initialize(*); end
def transform(context, data)
data
.then { |data| add_user(data) }
......
......@@ -4,8 +4,6 @@ module BulkImports
module Groups
module Transformers
class SubgroupToEntityTransformer
def initialize(*args); end
def transform(context, entry)
{
source_type: :group_entity,
......
......@@ -15,16 +15,78 @@ module BulkImports
attr_reader :context
# Fetch pipeline extractor.
# An extractor is defined either by instance `#extract(context)` method
# or by using `extractor` DSL.
#
# @example
# class MyPipeline
# extractor MyExtractor, foo: :bar
# end
#
# class MyPipeline
# def extract(context)
# puts 'Fetch some data'
# end
# end
#
# If pipeline implements instance method `extract` - use it
# and ignore class `extractor` method implementation.
def extractor
@extractor ||= instantiate(self.class.get_extractor)
@extractor ||= self.respond_to?(:extract) ? self : instantiate(self.class.get_extractor)
end
# Fetch pipeline transformers.
#
# A transformer can be defined using:
# - `transformer` class method
# - `transform` instance method
#
# Multiple transformers can be defined within a single
# pipeline and run sequentially for each record in the
# following order:
# - Transformers defined using `transformer` class method
# - Instance method `transform`
#
# Instance method `transform` is always the last to run.
#
# @example
# class MyPipeline
# transformer MyTransformerOne, foo: :bar
# transformer MyTransformerTwo, foo: :bar
#
# def transform(context, data)
# # perform transformation here
# end
# end
#
# In the example above `MyTransformerOne` is the first to run and
# the instance `#transform` method is the last.
def transformers
@transformers ||= self.class.transformers.map(&method(:instantiate))
@transformers << self if respond_to?(:transform) && @transformers.exclude?(self)
@transformers
end
# Fetch pipeline loader.
# A loader is defined either by instance method `#load(context, data)`
# or by using `loader` DSL.
#
# @example
# class MyPipeline
# loader MyLoader, foo: :bar
# end
#
# class MyPipeline
# def load(context, data)
# puts 'Load some data'
# end
# end
#
# If pipeline implements instance method `load` - use it
# and ignore class `loader` method implementation.
def loader
@loaders ||= instantiate(self.class.get_loader)
@loader ||= self.respond_to?(:load) ? self : instantiate(self.class.get_loader)
end
def pipeline
......@@ -32,7 +94,13 @@ module BulkImports
end
def instantiate(class_config)
class_config[:klass].new(class_config[:options])
options = class_config[:options]
if options
class_config[:klass].new(class_config[:options])
else
class_config[:klass].new
end
end
def abort_on_failure?
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Common::Loaders::EntityLoader do
describe '#load' do
it "creates entities for the given data" do
group = create(:group, path: "imported-group")
parent_entity = create(:bulk_import_entity, group: group, bulk_import: create(:bulk_import))
context = BulkImports::Pipeline::Context.new(parent_entity)
data = {
source_type: :group_entity,
source_full_path: "parent/subgroup",
destination_name: "subgroup",
destination_namespace: parent_entity.group.full_path,
parent_id: parent_entity.id
}
expect { subject.load(context, data) }.to change(BulkImports::Entity, :count).by(1)
subgroup_entity = BulkImports::Entity.last
expect(subgroup_entity.source_full_path).to eq 'parent/subgroup'
expect(subgroup_entity.destination_namespace).to eq 'imported-group'
expect(subgroup_entity.destination_name).to eq 'subgroup'
expect(subgroup_entity.parent_id).to eq parent_entity.id
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Groups::Loaders::LabelsLoader do
describe '#load' do
let(:user) { create(:user) }
let(:group) { create(:group) }
let(:entity) { create(:bulk_import_entity, group: group) }
let(:context) { BulkImports::Pipeline::Context.new(entity) }
let(:data) do
{
'title' => 'label',
'description' => 'description',
'color' => '#FFFFFF'
}
end
it 'creates the label' do
expect { subject.load(context, data) }.to change(Label, :count).by(1)
label = group.labels.first
expect(label.title).to eq(data['title'])
expect(label.description).to eq(data['description'])
expect(label.color).to eq(data['color'])
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe BulkImports::Groups::Loaders::MembersLoader do
describe '#load' do
let_it_be(:user_importer) { create(:user) }
let_it_be(:user_member) { create(:user) }
let_it_be(:group) { create(:group) }
let_it_be(:bulk_import) { create(:bulk_import, user: user_importer) }
let_it_be(:entity) { create(:bulk_import_entity, bulk_import: bulk_import, group: group) }
let_it_be(:context) { BulkImports::Pipeline::Context.new(entity) }
let_it_be(:data) do
{
'user_id' => user_member.id,
'created_by_id' => user_importer.id,
'access_level' => 30,
'created_at' => '2020-01-01T00:00:00Z',
'updated_at' => '2020-01-01T00:00:00Z',
'expires_at' => nil
}
end
it 'does nothing when there is no data' do
expect { subject.load(context, nil) }.not_to change(GroupMember, :count)
end
it 'creates the member' do
expect { subject.load(context, data) }.to change(GroupMember, :count).by(1)
member = group.members.last
expect(member.user).to eq(user_member)
expect(member.created_by).to eq(user_importer)
expect(member.access_level).to eq(30)
expect(member.created_at).to eq('2020-01-01T00:00:00Z')
expect(member.updated_at).to eq('2020-01-01T00:00:00Z')
expect(member.expires_at).to eq(nil)
end
end
end
......@@ -90,6 +90,24 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do
end
end
describe '#load' do
it 'creates the label' do
data = {
'title' => 'label',
'description' => 'description',
'color' => '#FFFFFF'
}
expect { subject.load(context, data) }.to change(Label, :count).by(1)
label = group.labels.first
expect(label.title).to eq(data['title'])
expect(label.description).to eq(data['description'])
expect(label.color).to eq(data['color'])
end
end
describe 'pipeline parts' do
it { expect(described_class).to include_module(BulkImports::Pipeline) }
it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) }
......@@ -110,9 +128,5 @@ RSpec.describe BulkImports::Groups::Pipelines::LabelsPipeline do
{ klass: BulkImports::Common::Transformers::ProhibitedAttributesTransformer, options: nil }
)
end
it 'has loaders' do
expect(described_class.get_loader).to eq(klass: BulkImports::Groups::Loaders::LabelsLoader, options: nil)
end
end
end
......@@ -37,6 +37,34 @@ RSpec.describe BulkImports::Groups::Pipelines::MembersPipeline do
end
end
describe '#load' do
it 'does nothing when there is no data' do
expect { subject.load(context, nil) }.not_to change(GroupMember, :count)
end
it 'creates the member' do
data = {
'user_id' => member_user1.id,
'created_by_id' => member_user2.id,
'access_level' => 30,
'created_at' => '2020-01-01T00:00:00Z',
'updated_at' => '2020-01-01T00:00:00Z',
'expires_at' => nil
}
expect { subject.load(context, data) }.to change(GroupMember, :count).by(1)
member = group.members.last
expect(member.user).to eq(member_user1)
expect(member.created_by).to eq(member_user2)
expect(member.access_level).to eq(30)
expect(member.created_at).to eq('2020-01-01T00:00:00Z')
expect(member.updated_at).to eq('2020-01-01T00:00:00Z')
expect(member.expires_at).to eq(nil)
end
end
describe 'pipeline parts' do
it { expect(described_class).to include_module(BulkImports::Pipeline) }
it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) }
......@@ -58,10 +86,6 @@ RSpec.describe BulkImports::Groups::Pipelines::MembersPipeline do
{ klass: BulkImports::Groups::Transformers::MemberAttributesTransformer, options: nil }
)
end
it 'has loaders' do
expect(described_class.get_loader).to eq(klass: BulkImports::Groups::Loaders::MembersLoader, options: nil)
end
end
def member_data(email:, has_next_page:, cursor: nil)
......
......@@ -3,9 +3,14 @@
require 'spec_helper'
RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do
let_it_be(:user) { create(:user) }
let_it_be(:group) { create(:group, path: 'group') }
let_it_be(:parent) { create(:group, name: 'imported-group', path: 'imported-group') }
let(:context) { BulkImports::Pipeline::Context.new(parent_entity) }
subject { described_class.new(context) }
describe '#run' do
let_it_be(:user) { create(:user) }
let(:parent) { create(:group, name: 'imported-group', path: 'imported-group') }
let!(:parent_entity) do
create(
:bulk_import_entity,
......@@ -14,8 +19,6 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do
)
end
let(:context) { BulkImports::Pipeline::Context.new(parent_entity) }
let(:subgroup_data) do
[
{
......@@ -25,8 +28,6 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do
]
end
subject { described_class.new(context) }
before do
allow_next_instance_of(BulkImports::Groups::Extractors::SubgroupsExtractor) do |extractor|
allow(extractor).to receive(:extract).and_return(subgroup_data)
......@@ -47,6 +48,29 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do
end
end
describe '#load' do
let(:parent_entity) { create(:bulk_import_entity, group: group, bulk_import: create(:bulk_import)) }
it 'creates entities for the given data' do
data = {
source_type: :group_entity,
source_full_path: 'parent/subgroup',
destination_name: 'subgroup',
destination_namespace: parent_entity.group.full_path,
parent_id: parent_entity.id
}
expect { subject.load(context, data) }.to change(BulkImports::Entity, :count).by(1)
subgroup_entity = BulkImports::Entity.last
expect(subgroup_entity.source_full_path).to eq 'parent/subgroup'
expect(subgroup_entity.destination_namespace).to eq 'group'
expect(subgroup_entity.destination_name).to eq 'subgroup'
expect(subgroup_entity.parent_id).to eq parent_entity.id
end
end
describe 'pipeline parts' do
it { expect(described_class).to include_module(BulkImports::Pipeline) }
it { expect(described_class).to include_module(BulkImports::Pipeline::Runner) }
......@@ -61,9 +85,5 @@ RSpec.describe BulkImports::Groups::Pipelines::SubgroupEntitiesPipeline do
{ klass: BulkImports::Groups::Transformers::SubgroupToEntityTransformer, options: nil }
)
end
it 'has loaders' do
expect(described_class.get_loader).to eq(klass: BulkImports::Common::Loaders::EntityLoader, options: nil)
end
end
end
......@@ -3,25 +3,25 @@
require 'spec_helper'
RSpec.describe BulkImports::Pipeline do
describe 'pipeline attributes' do
before do
stub_const('BulkImports::Extractor', Class.new)
stub_const('BulkImports::Transformer', Class.new)
stub_const('BulkImports::Loader', Class.new)
klass = Class.new do
include BulkImports::Pipeline
before do
stub_const('BulkImports::Extractor', Class.new)
stub_const('BulkImports::Transformer', Class.new)
stub_const('BulkImports::Loader', Class.new)
abort_on_failure!
klass = Class.new do
include BulkImports::Pipeline
extractor BulkImports::Extractor, { foo: :bar }
transformer BulkImports::Transformer, { foo: :bar }
loader BulkImports::Loader, { foo: :bar }
end
abort_on_failure!
stub_const('BulkImports::MyPipeline', klass)
extractor BulkImports::Extractor, foo: :bar
transformer BulkImports::Transformer, foo: :bar
loader BulkImports::Loader, foo: :bar
end
stub_const('BulkImports::MyPipeline', klass)
end
describe 'pipeline attributes' do
describe 'getters' do
it 'retrieves class attributes' do
expect(BulkImports::MyPipeline.get_extractor).to eq({ klass: BulkImports::Extractor, options: { foo: :bar } })
......@@ -29,6 +29,27 @@ RSpec.describe BulkImports::Pipeline do
expect(BulkImports::MyPipeline.get_loader).to eq({ klass: BulkImports::Loader, options: { foo: :bar } })
expect(BulkImports::MyPipeline.abort_on_failure?).to eq(true)
end
context 'when extractor and loader are defined within the pipeline' do
before do
klass = Class.new do
include BulkImports::Pipeline
def extract; end
def load; end
end
stub_const('BulkImports::AnotherPipeline', klass)
end
it 'returns itself when retrieving extractor & loader' do
pipeline = BulkImports::AnotherPipeline.new(nil)
expect(pipeline.send(:extractor)).to eq(pipeline)
expect(pipeline.send(:loader)).to eq(pipeline)
end
end
end
describe 'setters' do
......@@ -54,4 +75,46 @@ RSpec.describe BulkImports::Pipeline do
end
end
end
describe '#instantiate' do
context 'when options are present' do
it 'instantiates new object with options' do
expect(BulkImports::Extractor).to receive(:new).with(foo: :bar)
expect(BulkImports::Transformer).to receive(:new).with(foo: :bar)
expect(BulkImports::Loader).to receive(:new).with(foo: :bar)
pipeline = BulkImports::MyPipeline.new(nil)
pipeline.send(:extractor)
pipeline.send(:transformers)
pipeline.send(:loader)
end
end
context 'when options are missing' do
before do
klass = Class.new do
include BulkImports::Pipeline
extractor BulkImports::Extractor
transformer BulkImports::Transformer
loader BulkImports::Loader
end
stub_const('BulkImports::NoOptionsPipeline', klass)
end
it 'instantiates new object without options' do
expect(BulkImports::Extractor).to receive(:new).with(no_args)
expect(BulkImports::Transformer).to receive(:new).with(no_args)
expect(BulkImports::Loader).to receive(:new).with(no_args)
pipeline = BulkImports::NoOptionsPipeline.new(nil)
pipeline.send(:extractor)
pipeline.send(:transformers)
pipeline.send(:loader)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment