Commit c5faad8c authored by Kassio Borges's avatar Kassio Borges

Add logs about `BulkImports::Pipeline#run`

To make it easier to debug exceptions log what pipeline, and its
parts, is running and for what `BulkImports::Entity`.
parent 10332fe7
...@@ -6,33 +6,60 @@ module BulkImports ...@@ -6,33 +6,60 @@ module BulkImports
extend ActiveSupport::Concern extend ActiveSupport::Concern
included do included do
attr_reader :extractors, :transformers, :loaders private
def initialize def extractors
@extractors = self.class.extractors.map(&method(:instantiate)) @extractors ||= self.class.extractors.map(&method(:instantiate))
@transformers = self.class.transformers.map(&method(:instantiate)) end
@loaders = self.class.loaders.map(&method(:instantiate))
def transformers
@transformers ||= self.class.transformers.map(&method(:instantiate))
end
def loaders
@loaders ||= self.class.loaders.map(&method(:instantiate))
end
def pipeline_name
@pipeline ||= self.class.name
end
super def instantiate(class_config)
class_config[:klass].new(class_config[:options])
end
end end
def run(context) def run(context)
info(context, message: "Pipeline started", pipeline: pipeline_name)
extractors.each do |extractor| extractors.each do |extractor|
extractor.extract(context).each do |entry| extractor.extract(context).each do |entry|
info(context, extractor: extractor.class.name)
transformers.each do |transformer| transformers.each do |transformer|
info(context, transformer: transformer.class.name)
entry = transformer.transform(context, entry) entry = transformer.transform(context, entry)
end end
loaders.each do |loader| loaders.each do |loader|
info(context, loader: loader.class.name)
loader.load(context, entry) loader.load(context, entry)
end end
end end
end end
end end
def instantiate(class_config) private # rubocop:disable Lint/UselessAccessModifier
class_config[:klass].new(class_config[:options])
def info(context, extra = {})
logger.info({
entity: context.entity.id,
entity_type: context.entity.source_type
}.merge(extra))
end end
def logger
@logger ||= Gitlab::Import::Logger.build
end end
end end
end end
......
...@@ -39,7 +39,10 @@ RSpec.describe BulkImports::Pipeline::Runner do ...@@ -39,7 +39,10 @@ RSpec.describe BulkImports::Pipeline::Runner do
end end
it 'runs pipeline extractor, transformer, loader' do it 'runs pipeline extractor, transformer, loader' do
context = instance_double(BulkImports::Pipeline::Context) context = instance_double(
BulkImports::Pipeline::Context,
entity: instance_double(BulkImports::Entity, id: 1, source_type: 'group')
)
entries = [{ foo: :bar }] entries = [{ foo: :bar }]
expect_next_instance_of(BulkImports::Extractor) do |extractor| expect_next_instance_of(BulkImports::Extractor) do |extractor|
...@@ -54,6 +57,17 @@ RSpec.describe BulkImports::Pipeline::Runner do ...@@ -54,6 +57,17 @@ RSpec.describe BulkImports::Pipeline::Runner do
expect(loader).to receive(:load).with(context, entries.first) expect(loader).to receive(:load).with(context, entries.first)
end end
expect_next_instance_of(Gitlab::Import::Logger) do |logger|
expect(logger).to receive(:info)
.with(message: "Pipeline started", pipeline: 'BulkImports::MyPipeline', entity: 1, entity_type: 'group')
expect(logger).to receive(:info)
.with(entity: 1, entity_type: 'group', extractor: 'BulkImports::Extractor')
expect(logger).to receive(:info)
.with(entity: 1, entity_type: 'group', transformer: 'BulkImports::Transformer')
expect(logger).to receive(:info)
.with(entity: 1, entity_type: 'group', loader: 'BulkImports::Loader')
end
BulkImports::MyPipeline.new.run(context) BulkImports::MyPipeline.new.run(context)
end end
end end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment