Commit b9774b2c authored by Stan Hu's avatar Stan Hu

Merge branch '246857-requirements-management-import-csv' into 'master'

Add service to import Requirements from a CSV file

See merge request gitlab-org/gitlab!46361
parents 3dece468 237d3446
# frozen_string_literal: true
module Issuable
module ImportCsv
class BaseService
def initialize(user, project, csv_io)
@user = user
@project = project
@csv_io = csv_io
@results = { success: 0, error_lines: [], parse_error: false }
end
def execute
process_csv
email_results_to_user
@results
end
private
def process_csv
with_csv_lines.each do |row, line_no|
issuable_attributes = {
title: row[:title],
description: row[:description]
}
if create_issuable(issuable_attributes).persisted?
@results[:success] += 1
else
@results[:error_lines].push(line_no)
end
end
rescue ArgumentError, CSV::MalformedCSVError
@results[:parse_error] = true
end
def with_csv_lines
csv_data = @csv_io.open(&:read).force_encoding(Encoding::UTF_8)
verify_headers!(csv_data)
csv_parsing_params = {
col_sep: detect_col_sep(csv_data.lines.first),
headers: true,
header_converters: :symbol
}
CSV.new(csv_data, csv_parsing_params).each.with_index(2)
end
def verify_headers!(data)
headers = data.lines.first.downcase
return if headers.include?('title') && headers.include?('description')
raise CSV::MalformedCSVError
end
def detect_col_sep(header)
if header.include?(",")
","
elsif header.include?(";")
";"
elsif header.include?("\t")
"\t"
else
raise CSV::MalformedCSVError
end
end
def create_issuable(attributes)
create_issuable_class.new(@project, @user, attributes).execute
end
def email_results_to_user
# defined in ImportCsvService
end
def create_issuable_class
# defined in ImportCsvService
end
end
end
end
# frozen_string_literal: true
module Issues
class ImportCsvService
def initialize(user, project, csv_io)
@user = user
@project = project
@csv_io = csv_io
@results = { success: 0, error_lines: [], parse_error: false }
end
class ImportCsvService < Issuable::ImportCsv::BaseService
def execute
record_import_attempt
process_csv
email_results_to_user
@results
super
end
private
def record_import_attempt
Issues::CsvImport.create!(user: @user, project: @project)
def email_results_to_user
Notify.import_issues_csv_email(@user.id, @project.id, @results).deliver_later
end
def process_csv
csv_data = @csv_io.open(&:read).force_encoding(Encoding::UTF_8)
csv_parsing_params = {
col_sep: detect_col_sep(csv_data.lines.first),
headers: true,
header_converters: :symbol
}
CSV.new(csv_data, csv_parsing_params).each.with_index(2) do |row, line_no|
issue_attributes = {
title: row[:title],
description: row[:description]
}
issue = Issues::CreateService.new(@project, @user, issue_attributes).execute
private
if issue.persisted?
@results[:success] += 1
else
@results[:error_lines].push(line_no)
end
end
rescue ArgumentError, CSV::MalformedCSVError
@results[:parse_error] = true
def create_issuable_class
Issues::CreateService
end
def email_results_to_user
Notify.import_issues_csv_email(@user.id, @project.id, @results).deliver_later
end
def detect_col_sep(header)
if header.include?(",")
","
elsif header.include?(";")
";"
elsif header.include?("\t")
"\t"
else
raise CSV::MalformedCSVError
end
def record_import_attempt
Issues::CsvImport.create!(user: @user, project: @project)
end
end
end
# frozen_string_literal: true
module RequirementsManagement
class ImportCsvService < ::Issuable::ImportCsv::BaseService
private
def create_issuable_class
RequirementsManagement::CreateRequirementService
end
end
end
---
title: Add service to import Requirements from a CSV file
merge_request: 46361
author:
type: added
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe RequirementsManagement::ImportCsvService do
let_it_be_with_refind(:project) { create(:project) }
let_it_be(:user) { create(:user) }
let(:service) do
uploader = FileUploader.new(project)
uploader.store!(file)
described_class.new(user, project, uploader)
end
shared_examples 'resource not available' do
it 'raises an error' do
expect { service.execute }.to raise_error(Gitlab::Access::AccessDeniedError)
end
end
before do
project.add_reporter(user)
stub_licensed_features(requirements: true)
end
context 'when user can create requirements' do
include_examples 'issuable import csv service', 'requirement' do
let(:issuables) { project.requirements }
let(:email_method) { nil }
end
end
context 'when user cannot create requirements' do
let(:file) { fixture_file_upload('spec/fixtures/csv_comma.csv') }
before do
project.add_guest(user)
end
it_behaves_like 'resource not available'
end
context 'when requirements feature is not available' do
let(:file) { fixture_file_upload('spec/fixtures/csv_comma.csv') }
before do
stub_licensed_features(requirements: false)
end
it_behaves_like 'resource not available'
end
end
Issue in 中文,Test description
"Hello","World"
"Title with quote""",Description
......@@ -5,126 +5,15 @@ require 'spec_helper'
RSpec.describe Issues::ImportCsvService do
let(:project) { create(:project) }
let(:user) { create(:user) }
subject do
let(:service) do
uploader = FileUploader.new(project)
uploader.store!(file)
described_class.new(user, project, uploader).execute
end
shared_examples_for 'an issue importer' do
it 'records the import attempt' do
expect { subject }
.to change { Issues::CsvImport.where(project: project, user: user).count }
.by 1
end
described_class.new(user, project, uploader)
end
describe '#execute' do
context 'invalid file' do
let(:file) { fixture_file_upload('spec/fixtures/banana_sample.gif') }
it 'returns invalid file error' do
expect(Notify).to receive_message_chain(:import_issues_csv_email, :deliver_later)
expect(subject[:success]).to eq(0)
expect(subject[:parse_error]).to eq(true)
end
it_behaves_like 'an issue importer'
end
context 'with a file generated by Gitlab CSV export' do
let(:file) { fixture_file_upload('spec/fixtures/csv_gitlab_export.csv') }
it 'imports the CSV without errors' do
expect(Notify).to receive_message_chain(:import_issues_csv_email, :deliver_later)
expect(subject[:success]).to eq(4)
expect(subject[:error_lines]).to eq([])
expect(subject[:parse_error]).to eq(false)
end
it 'correctly sets the issue attributes' do
expect { subject }.to change { project.issues.count }.by 4
expect(project.issues.reload.last).to have_attributes(
title: 'Test Title',
description: 'Test Description'
)
end
it_behaves_like 'an issue importer'
end
context 'comma delimited file' do
let(:file) { fixture_file_upload('spec/fixtures/csv_comma.csv') }
it 'imports CSV without errors' do
expect(Notify).to receive_message_chain(:import_issues_csv_email, :deliver_later)
expect(subject[:success]).to eq(3)
expect(subject[:error_lines]).to eq([])
expect(subject[:parse_error]).to eq(false)
end
it 'correctly sets the issue attributes' do
expect { subject }.to change { project.issues.count }.by 3
expect(project.issues.reload.last).to have_attributes(
title: 'Title with quote"',
description: 'Description'
)
end
it_behaves_like 'an issue importer'
end
context 'tab delimited file with error row' do
let(:file) { fixture_file_upload('spec/fixtures/csv_tab.csv') }
it 'imports CSV with some error rows' do
expect(Notify).to receive_message_chain(:import_issues_csv_email, :deliver_later)
expect(subject[:success]).to eq(2)
expect(subject[:error_lines]).to eq([3])
expect(subject[:parse_error]).to eq(false)
end
it 'correctly sets the issue attributes' do
expect { subject }.to change { project.issues.count }.by 2
expect(project.issues.reload.last).to have_attributes(
title: 'Hello',
description: 'World'
)
end
it_behaves_like 'an issue importer'
end
context 'semicolon delimited file with CRLF' do
let(:file) { fixture_file_upload('spec/fixtures/csv_semicolon.csv') }
it 'imports CSV with a blank row' do
expect(Notify).to receive_message_chain(:import_issues_csv_email, :deliver_later)
expect(subject[:success]).to eq(3)
expect(subject[:error_lines]).to eq([4])
expect(subject[:parse_error]).to eq(false)
end
it 'correctly sets the issue attributes' do
expect { subject }.to change { project.issues.count }.by 3
expect(project.issues.reload.last).to have_attributes(
title: 'Hello',
description: 'World'
)
end
it_behaves_like 'an issue importer'
end
include_examples 'issuable import csv service', 'issue' do
let(:issuables) { project.issues }
let(:email_method) { :import_issues_csv_email }
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.shared_examples 'issuable import csv service' do |issuable_type|
let_it_be_with_refind(:project) { create(:project) }
let_it_be(:user) { create(:user) }
subject { service.execute }
shared_examples_for 'an issuable importer' do
if issuable_type == 'issue'
it 'records the import attempt if resource is an issue' do
expect { subject }
.to change { Issues::CsvImport.where(project: project, user: user).count }
.by 1
end
end
end
shared_examples_for 'importer with email notification' do
if issuable_type == 'issue'
it 'notifies user of import result' do
expect(Notify).to receive_message_chain(email_method, :deliver_later)
subject
end
end
end
describe '#execute' do
context 'invalid file' do
let(:file) { fixture_file_upload('spec/fixtures/banana_sample.gif') }
it 'returns invalid file error' do
expect(subject[:success]).to eq(0)
expect(subject[:parse_error]).to eq(true)
end
it_behaves_like 'importer with email notification'
it_behaves_like 'an issuable importer'
end
context 'file without headers' do
let(:file) { fixture_file_upload('spec/fixtures/csv_no_headers.csv') }
it 'returns invalid file error' do
expect(subject[:success]).to eq(0)
expect(subject[:parse_error]).to eq(true)
end
it_behaves_like 'importer with email notification'
it_behaves_like 'an issuable importer'
end
context 'with a file generated by Gitlab CSV export' do
let(:file) { fixture_file_upload('spec/fixtures/csv_gitlab_export.csv') }
it 'imports the CSV without errors' do
expect(subject[:success]).to eq(4)
expect(subject[:error_lines]).to eq([])
expect(subject[:parse_error]).to eq(false)
end
it 'correctly sets the issuable attributes' do
expect { subject }.to change { issuables.count }.by 4
expect(issuables.reload.last).to have_attributes(
title: 'Test Title',
description: 'Test Description'
)
end
it_behaves_like 'importer with email notification'
it_behaves_like 'an issuable importer'
end
context 'comma delimited file' do
let(:file) { fixture_file_upload('spec/fixtures/csv_comma.csv') }
it 'imports CSV without errors' do
expect(subject[:success]).to eq(3)
expect(subject[:error_lines]).to eq([])
expect(subject[:parse_error]).to eq(false)
end
it 'correctly sets the issuable attributes' do
expect { subject }.to change { issuables.count }.by 3
expect(issuables.reload.last).to have_attributes(
title: 'Title with quote"',
description: 'Description'
)
end
it_behaves_like 'importer with email notification'
it_behaves_like 'an issuable importer'
end
context 'tab delimited file with error row' do
let(:file) { fixture_file_upload('spec/fixtures/csv_tab.csv') }
it 'imports CSV with some error rows' do
expect(subject[:success]).to eq(2)
expect(subject[:error_lines]).to eq([3])
expect(subject[:parse_error]).to eq(false)
end
it 'correctly sets the issuable attributes' do
expect { subject }.to change { issuables.count }.by 2
expect(issuables.reload.last).to have_attributes(
title: 'Hello',
description: 'World'
)
end
it_behaves_like 'importer with email notification'
it_behaves_like 'an issuable importer'
end
context 'semicolon delimited file with CRLF' do
let(:file) { fixture_file_upload('spec/fixtures/csv_semicolon.csv') }
it 'imports CSV with a blank row' do
expect(subject[:success]).to eq(3)
expect(subject[:error_lines]).to eq([4])
expect(subject[:parse_error]).to eq(false)
end
it 'correctly sets the issuable attributes' do
expect { subject }.to change { issuables.count }.by 3
expect(issuables.reload.last).to have_attributes(
title: 'Hello',
description: 'World'
)
end
it_behaves_like 'importer with email notification'
it_behaves_like 'an issuable importer'
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment