Skip to content

G
gitlab-ce
Commit b225165e authored by Alper Akgun's avatar Alper Akgun
Browse files
Options

Merge branch '351921-optimize-coverage-report-worker' into 'master' 

Utilize Nokogiri streaming capabilities for CoverageReportWorker

See merge request gitlab-org/gitlab!79866
parents 31fbab7a 66c1b902
Hide whitespace changes
Inline Side-by-side
Showing with 1035 additions and 813 deletions
config/feature_flags/development/use_cobertura_sax_parser.yml 0 → 100644
View file @ b225165e
---
name: use_cobertura_sax_parser
introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/79866
rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/352579
milestone: '14.9'
type: development
group: group::memory
default_enabled: false
lib/gitlab/ci/parsers/coverage/cobertura.rb
View file @ b225165e
...@@ -8,140 +8,12 @@ module Gitlab ...@@ -8,140 +8,12 @@ module Gitlab
InvalidXMLError = Class.new(Gitlab::Ci::Parsers::ParserError) InvalidXMLError = Class.new(Gitlab::Ci::Parsers::ParserError)
InvalidLineInformationError = Class.new(Gitlab::Ci::Parsers::ParserError) InvalidLineInformationError = Class.new(Gitlab::Ci::Parsers::ParserError)
GO_SOURCE_PATTERN = '/usr/local/go/src'
MAX_SOURCES = 100
def parse!(xml_data, coverage_report, project_path: nil, worktree_paths: nil) def parse!(xml_data, coverage_report, project_path: nil, worktree_paths: nil)
root = Hash.from_xml(xml_data) if Feature.enabled?(:use_cobertura_sax_parser, default_enabled: :yaml)
Nokogiri::XML::SAX::Parser.new(SaxDocument.new(coverage_report, project_path, worktree_paths)).parse(xml_data)
context = { else
project_path: project_path, DomParser.new.parse(xml_data, coverage_report, project_path, worktree_paths)
paths: worktree_paths&.to_set,
sources: []
}
parse_all(root, coverage_report, context)
rescue Nokogiri::XML::SyntaxError
raise InvalidXMLError, "XML parsing failed"
end
private
def parse_all(root, coverage_report, context)
return unless root.present?
root.each do |key, value|
parse_node(key, value, coverage_report, context)
end
end
def parse_node(key, value, coverage_report, context)
if key == 'sources' && value && value['source'].present?
parse_sources(value['source'], context)
elsif key == 'package'
Array.wrap(value).each do |item|
parse_package(item, coverage_report, context)
end
elsif key == 'class'
# This means the cobertura XML does not have classes within package nodes.
# This is possible in some cases like in simple JS project structures
# running Jest.
Array.wrap(value).each do |item|
parse_class(item, coverage_report, context)
end
elsif value.is_a?(Hash)
parse_all(value, coverage_report, context)
elsif value.is_a?(Array)
value.each do |item|
parse_all(item, coverage_report, context)
end
end
end
def parse_sources(sources, context)
return unless context[:project_path] && context[:paths]
sources = Array.wrap(sources)
# TODO: Go cobertura has a different format with how their packages
# are included in the filename. So we can't rely on the sources.
# We'll deal with this later.
return if sources.include?(GO_SOURCE_PATTERN)
sources.each do |source|
source = build_source_path(source, context)
context[:sources] << source if source.present?
end
end
def build_source_path(source, context)
# | raw source | extracted |
# |-----------------------------|------------|
# | /builds/foo/test/SampleLib/ | SampleLib/ |
# | /builds/foo/test/something | something |
# | /builds/foo/test/ | nil |
# | /builds/foo/test | nil |
source.split("#{context[:project_path]}/", 2)[1]
end
def parse_package(package, coverage_report, context)
classes = package.dig('classes', 'class')
return unless classes.present?
matched_filenames = Array.wrap(classes).map do |item|
parse_class(item, coverage_report, context)
end end
# Remove these filenames from the paths to avoid conflict
# with other packages that may contain the same class filenames
remove_matched_filenames(matched_filenames, context)
end
def remove_matched_filenames(filenames, context)
return unless context[:paths]
filenames.each { |f| context[:paths].delete(f) }
end
def parse_class(file, coverage_report, context)
return unless file["filename"].present? && file["lines"].present?
parsed_lines = parse_lines(file["lines"])
filename = determine_filename(file["filename"], context)
coverage_report.add_file(filename, Hash[parsed_lines]) if filename
filename
end
def parse_lines(lines)
line_array = Array.wrap(lines["line"])
line_array.map do |line|
# Using `Integer()` here to raise exception on invalid values
[Integer(line["number"]), Integer(line["hits"])]
end
rescue StandardError
raise InvalidLineInformationError, "Line information had invalid values"
end
def determine_filename(filename, context)
return filename unless context[:sources].any?
full_filename = nil
context[:sources].each_with_index do |source, index|
break if index >= MAX_SOURCES
break if full_filename = check_source(source, filename, context)
end
full_filename
end
def check_source(source, filename, context)
full_path = File.join(source, filename)
return full_path if context[:paths].include?(full_path)
end end
end end
end end
......
lib/gitlab/ci/parsers/coverage/dom_parser.rb 0 → 100644
View file @ b225165e
# frozen_string_literal: true
module Gitlab
module Ci
module Parsers
module Coverage
class DomParser
GO_SOURCE_PATTERN = '/usr/local/go/src'
MAX_SOURCES = 100
def parse(xml_data, coverage_report, project_path, worktree_paths)
root = Hash.from_xml(xml_data)
context = {
project_path: project_path,
paths: worktree_paths&.to_set,
sources: []
}
parse_all(root, coverage_report, context)
rescue Nokogiri::XML::SyntaxError
raise Cobertura::InvalidXMLError, "XML parsing failed"
end
private
def parse_all(root, coverage_report, context)
return unless root.present?
root.each do |key, value|
parse_node(key, value, coverage_report, context)
end
end
def parse_node(key, value, coverage_report, context)
if key == 'sources' && value && value['source'].present?
parse_sources(value['source'], context)
elsif key == 'package'
Array.wrap(value).each do |item|
parse_package(item, coverage_report, context)
end
elsif key == 'class'
# This means the cobertura XML does not have classes within package nodes.
# This is possible in some cases like in simple JS project structures
# running Jest.
Array.wrap(value).each do |item|
parse_class(item, coverage_report, context)
end
elsif value.is_a?(Hash)
parse_all(value, coverage_report, context)
elsif value.is_a?(Array)
value.each do |item|
parse_all(item, coverage_report, context)
end
end
end
def parse_sources(sources, context)
return unless context[:project_path] && context[:paths]
sources = Array.wrap(sources)
# TODO: Go cobertura has a different format with how their packages
# are included in the filename. So we can't rely on the sources.
# We'll deal with this later.
return if sources.include?(GO_SOURCE_PATTERN)
sources.each do |source|
source = build_source_path(source, context)
context[:sources] << source if source.present?
end
end
def build_source_path(source, context)
# | raw source | extracted |
# |-----------------------------|------------|
# | /builds/foo/test/SampleLib/ | SampleLib/ |
# | /builds/foo/test/something | something |
# | /builds/foo/test/ | nil |
# | /builds/foo/test | nil |
source.split("#{context[:project_path]}/", 2)[1]
end
def parse_package(package, coverage_report, context)
classes = package.dig('classes', 'class')
return unless classes.present?
matched_filenames = Array.wrap(classes).map do |item|
parse_class(item, coverage_report, context)
end
# Remove these filenames from the paths to avoid conflict
# with other packages that may contain the same class filenames
remove_matched_filenames(matched_filenames, context)
end
def remove_matched_filenames(filenames, context)
return unless context[:paths]
filenames.each { |f| context[:paths].delete(f) }
end
def parse_class(file, coverage_report, context)
return unless file["filename"].present? && file["lines"].present?
parsed_lines = parse_lines(file["lines"])
filename = determine_filename(file["filename"], context)
coverage_report.add_file(filename, Hash[parsed_lines]) if filename
filename
end
def parse_lines(lines)
line_array = Array.wrap(lines["line"])
line_array.map do |line|
# Using `Integer()` here to raise exception on invalid values
[Integer(line["number"]), Integer(line["hits"])]
end
rescue StandardError
raise Cobertura::InvalidLineInformationError, "Line information had invalid values"
end
def determine_filename(filename, context)
return filename unless context[:sources].any?
full_filename = nil
context[:sources].each_with_index do |source, index|
break if index >= MAX_SOURCES
break if full_filename = check_source(source, filename, context)
end
full_filename
end
def check_source(source, filename, context)
full_path = File.join(source, filename)
return full_path if context[:paths].include?(full_path)
end
end
end
end
end
end
lib/gitlab/ci/parsers/coverage/sax_document.rb 0 → 100644
View file @ b225165e
# frozen_string_literal: true
module Gitlab
module Ci
module Parsers
module Coverage
class SaxDocument < Nokogiri::XML::SAX::Document
GO_SOURCE_PATTERN = '/usr/local/go/src'
MAX_SOURCES = 100
def initialize(coverage_report, project_path, worktree_paths)
@coverage_report = coverage_report
@project_path = project_path
@paths = worktree_paths&.to_set
@matched_filenames = []
@parsed_lines = []
@sources = []
end
def error(error)
raise Cobertura::InvalidXMLError, "XML parsing failed with error: #{error}"
end
def start_element(node_name, attrs = [])
return unless node_name
self.node_name = node_name
node_attrs = Hash[attrs]
if node_name == 'class' && node_attrs["filename"].present?
self.filename = determine_filename(node_attrs["filename"])
self.matched_filenames << filename if filename
elsif node_name == 'line'
self.parsed_lines << parse_line(node_attrs)
end
end
def characters(node_content)
if node_name == 'source'
parse_source(node_content)
end
end
def end_element(node_name)
if node_name == "package"
remove_matched_filenames
elsif node_name == "class" && filename && parsed_lines.present?
coverage_report.add_file(filename, Hash[parsed_lines])
self.filename = nil
self.parsed_lines = []
end
end
private
attr_accessor :coverage_report, :project_path, :paths, :sources, :node_name, :filename, :parsed_lines, :matched_filenames
def parse_line(line)
[Integer(line["number"]), Integer(line["hits"])]
rescue StandardError
raise Cobertura::InvalidLineInformationError, "Line information had invalid values"
end
def parse_source(node)
return unless project_path && paths && !node.include?(GO_SOURCE_PATTERN)
source = build_source_path(node)
self.sources << source if source.present?
end
def build_source_path(node)
# | raw source | extracted |
# |-----------------------------|------------|
# | /builds/foo/test/SampleLib/ | SampleLib/ |
# | /builds/foo/test/something | something |
# | /builds/foo/test/ | nil |
# | /builds/foo/test | nil |
node.split("#{project_path}/", 2)[1]
end
def remove_matched_filenames
return unless paths
matched_filenames.each { |f| paths.delete(f) }
end
def determine_filename(filename)
return filename unless sources.any?
full_filename = nil
sources.each_with_index do |source, index|
break if index >= MAX_SOURCES
break if full_filename = check_source(source, filename)
end
full_filename
end
def check_source(source, filename)
full_path = File.join(source, filename)
return full_path if paths.include?(full_path)
end
end
end
end
end
end
spec/lib/gitlab/ci/parsers/coverage/cobertura_spec.rb
View file @ b225165e
# frozen_string_literal: true # frozen_string_literal: true
require 'fast_spec_helper' require 'spec_helper'
RSpec.describe Gitlab::Ci::Parsers::Coverage::Cobertura do RSpec.describe Gitlab::Ci::Parsers::Coverage::Cobertura do
describe '#parse!' do let(:xml_data) { double }
subject(:parse_report) { described_class.new.parse!(cobertura, coverage_report, project_path: project_path, worktree_paths: paths) } let(:coverage_report) { double }
let(:project_path) { double }
let(:paths) { double }
let(:coverage_report) { Gitlab::Ci::Reports::CoverageReports.new } subject(:parse_report) { described_class.new.parse!(xml_data, coverage_report, project_path: project_path, worktree_paths: paths) }
let(:project_path) { 'foo/bar' }
let(:paths) { ['app/user.rb'] }
let(:cobertura) do context 'when use_cobertura_sax_parser feature flag is enabled' do
<<~EOF before do
<coverage> stub_feature_flags(use_cobertura_sax_parser: true)
#{sources_xml}
#{classes_xml}
</coverage>
EOF
end
context 'when data is Cobertura style XML' do
shared_examples_for 'ignoring sources, project_path, and worktree_paths' do
context 'when there is no <class>' do
let(:classes_xml) { '' }
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'when there is a single <class>' do
context 'with no lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'with a single line' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2 } })
end
end
context 'without a package parent' do
let(:classes_xml) do
<<~EOF
<packages>
<class filename="app.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</packages>
EOF
end
it 'parses XML and returns a single file with coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2 } })
end
end
context 'with multiple lines and methods info' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2, 2 => 0 } })
end
end
end
context 'when there are multiple <class>' do
context 'without a package parent' do
let(:classes_xml) do
<<~EOF
<packages>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
</lines></class>
<class filename="foo.rb"><methods/><lines>
<line number="6" hits="1"/>
</lines></class>
</packages>
EOF
end
it 'parses XML and returns coverage information per class' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2 }, 'foo.rb' => { 6 => 1 } })
end
end
context 'with the same filename and different lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="app.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with merged coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2, 2 => 0, 6 => 1, 7 => 1 } })
end
end
context 'with the same filename and lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="1"/>
<line number="2" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with summed-up coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 3, 2 => 1 } })
end
end
context 'with missing filename' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and ignores class with missing name' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2, 2 => 0 } })
end
end
context 'with invalid line information' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="app.rb"><methods/><lines>
<line null="test" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'raises an error' do
expect { parse_report }.to raise_error(described_class::InvalidLineInformationError)
end
end
end
end
context 'when there is no <sources>' do
let(:sources_xml) { '' }
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'when there is an empty <sources>' do
let(:sources_xml) { '<sources />' }
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'when there is a <sources>' do
context 'and has a single source with a pattern for Go projects' do
let(:project_path) { 'local/go' } # Make sure we're not making false positives
let(:sources_xml) do
<<~EOF
<sources>
<source>/usr/local/go/src</source>
</sources>
EOF
end
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'and has multiple sources with a pattern for Go projects' do
let(:project_path) { 'local/go' } # Make sure we're not making false positives
let(:sources_xml) do
<<~EOF
<sources>
<source>/usr/local/go/src</source>
<source>/go/src</source>
</sources>
EOF
end
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'and has a single source but already is at the project root path' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/#{project_path}</source>
</sources>
EOF
end
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'and has multiple sources but already are at the project root path' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/#{project_path}/</source>
<source>builds/somewhere/#{project_path}</source>
</sources>
EOF
end
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'and has a single source that is not at the project root path' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/#{project_path}/app</source>
</sources>
EOF
end
context 'when there is no <class>' do
let(:classes_xml) { '' }
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'when there is a single <class>' do
context 'with no lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'with a single line but the filename cannot be determined based on extracted source and worktree paths' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="member.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({}) allow_next_instance_of(Nokogiri::XML::SAX::Parser) do |document|
end allow(document).to receive(:parse)
end
context 'with a single line' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2 } })
end
end
context 'with multiple lines and methods info' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0 } })
end
end
end
context 'when there are multiple <class>' do
context 'with the same filename but the filename cannot be determined based on extracted source and worktree paths' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="member.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="member.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'without a parent package' do
let(:classes_xml) do
<<~EOF
<packages>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="user.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</packages>
EOF
end
it 'parses XML and returns coverage information with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0, 6 => 1, 7 => 1 } })
end
end
context 'with the same filename and different lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="user.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with merged coverage, and with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0, 6 => 1, 7 => 1 } })
end
end
context 'with the same filename and lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="1"/>
<line number="2" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with summed-up coverage, and with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 3, 2 => 1 } })
end
end
context 'with missing filename' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and ignores class with missing name' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0 } })
end
end
context 'with filename that cannot be determined based on extracted source and worktree paths' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="member.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and ignores class with undetermined filename' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0 } })
end
end
context 'with invalid line information' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="user.rb"><methods/><lines>
<line null="test" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'raises an error' do
expect { parse_report }.to raise_error(described_class::InvalidLineInformationError)
end
end
end
end
context 'and has multiple sources that are not at the project root path' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/#{project_path}/app1/</source>
<source>builds/#{project_path}/app2/</source>
</sources>
EOF
end
context 'and a class filename is available under multiple extracted sources' do
let(:paths) { ['app1/user.rb', 'app2/user.rb'] }
let(:classes_xml) do
<<~EOF
<package name="app1">
<classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes>
</package>
<package name="app2">
<classes>
<class filename="user.rb"><lines>
<line number="2" hits="3"/>
</lines></class>
</classes>
</package>
EOF
end
it 'parses XML and returns the files with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({
'app1/user.rb' => { 1 => 2 },
'app2/user.rb' => { 2 => 3 }
})
end
end
context 'and a class filename is available under one of the extracted sources' do
let(:paths) { ['app1/member.rb', 'app2/user.rb', 'app2/pet.rb'] }
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with the filename relative to project root using the extracted source where it is first found under' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app2/user.rb' => { 1 => 2 } })
end
end
context 'and a class filename is not found under any of the extracted sources' do
let(:paths) { ['app1/member.rb', 'app2/pet.rb'] }
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'and a class filename is not found under any of the extracted sources within the iteratable limit' do
let(:paths) { ['app2/user.rb'] }
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="record.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
before do
stub_const("#{described_class}::MAX_SOURCES", 1)
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
end
end
shared_examples_for 'non-smart parsing' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/foo/bar/app</source>
</sources>
EOF
end
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns filenames unchanged just as how they are found in the class node' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'user.rb' => { 1 => 2 } })
end
end end
end
context 'when project_path is not present' do it 'uses Sax parser' do
let(:project_path) { nil } expect(Gitlab::Ci::Parsers::Coverage::SaxDocument).to receive(:new)
let(:paths) { ['app/user.rb'] }
it_behaves_like 'non-smart parsing' parse_report
end end
end
context 'when worktree_paths is not present' do context 'when use_cobertura_sax_parser feature flag is disabled' do
let(:project_path) { 'foo/bar' } before do
let(:paths) { nil } stub_feature_flags(use_cobertura_sax_parser: false)
it_behaves_like 'non-smart parsing' allow_next_instance_of(Gitlab::Ci::Parsers::Coverage::DomParser) do |parser|
allow(parser).to receive(:parse)
end end
end end
context 'when data is not Cobertura style XML' do it 'uses Dom parser' do
let(:cobertura) { { coverage: '12%' }.to_json } expect(Gitlab::Ci::Parsers::Coverage::DomParser).to receive(:new)
it 'raises an error' do parse_report
expect { parse_report }.to raise_error(described_class::InvalidXMLError)
end
end end
end end
end end
spec/lib/gitlab/ci/parsers/coverage/dom_parser_spec.rb 0 → 100644
View file @ b225165e
# frozen_string_literal: true
require 'fast_spec_helper'
require 'support/shared_examples/lib/gitlab/ci/parsers/coverage/cobertura_xml_shared_examples'
RSpec.describe Gitlab::Ci::Parsers::Coverage::DomParser do
subject(:parse_report) { described_class.new.parse(cobertura, coverage_report, project_path, paths) }
include_examples 'parse cobertura xml'
end
spec/lib/gitlab/ci/parsers/coverage/sax_document_spec.rb 0 → 100644
View file @ b225165e
# frozen_string_literal: true
require 'fast_spec_helper'
require 'support/shared_examples/lib/gitlab/ci/parsers/coverage/cobertura_xml_shared_examples'
RSpec.describe Gitlab::Ci::Parsers::Coverage::SaxDocument do
subject(:parse_report) { Nokogiri::XML::SAX::Parser.new(described_class.new(coverage_report, project_path, paths)).parse(cobertura) }
include_examples 'parse cobertura xml'
end
spec/support/shared_examples/lib/gitlab/ci/parsers/coverage/cobertura_xml_shared_examples.rb 0 → 100644
View file @ b225165e
# frozen_string_literal: true
RSpec.shared_examples 'parse cobertura xml' do
describe '#parse!' do
let(:coverage_report) { Gitlab::Ci::Reports::CoverageReports.new }
let(:project_path) { 'foo/bar' }
let(:paths) { ['app/user.rb'] }
let(:cobertura) do
<<~EOF
<coverage>
#{sources_xml}
#{classes_xml}
</coverage>
EOF
end
context 'when data is Cobertura style XML' do
shared_examples_for 'ignoring sources, project_path, and worktree_paths' do
context 'when there is no <class>' do
let(:classes_xml) { '' }
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'when there is a single <class>' do
context 'with no lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'with a single line' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2 } })
end
end
context 'without a package parent' do
let(:classes_xml) do
<<~EOF
<packages>
<class filename="app.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</packages>
EOF
end
it 'parses XML and returns a single file with coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2 } })
end
end
context 'with multiple lines and methods info' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2, 2 => 0 } })
end
end
end
context 'when there are multiple packages' do
let(:cobertura) do
<<~EOF
<coverage>
<packages><package name="app1"><classes>
<class filename="app1.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
<packages><package name="app2"><classes>
<class filename="app2.rb"><lines>
<line number="11" hits="3"/>
</lines></class>
</classes></package></packages>
</coverage>
EOF
end
it 'parses XML and returns coverage information per class' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app1.rb' => { 1 => 2 }, 'app2.rb' => { 11 => 3 } })
end
end
context 'when there are multiple <class>' do
context 'without a package parent' do
let(:classes_xml) do
<<~EOF
<packages>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
</lines></class>
<class filename="foo.rb"><methods/><lines>
<line number="6" hits="1"/>
</lines></class>
</packages>
EOF
end
it 'parses XML and returns coverage information per class' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2 }, 'foo.rb' => { 6 => 1 } })
end
end
context 'with the same filename and different lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="app.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with merged coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2, 2 => 0, 6 => 1, 7 => 1 } })
end
end
context 'with the same filename and lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="1"/>
<line number="2" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with summed-up coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 3, 2 => 1 } })
end
end
context 'with missing filename' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and ignores class with missing name' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app.rb' => { 1 => 2, 2 => 0 } })
end
end
context 'with invalid line information' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="app.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="app.rb"><methods/><lines>
<line null="test" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'raises an error' do
expect { parse_report }.to raise_error(Gitlab::Ci::Parsers::Coverage::Cobertura::InvalidLineInformationError)
end
end
end
end
context 'when there is no <sources>' do
let(:sources_xml) { '' }
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'when there is an empty <sources>' do
let(:sources_xml) { '<sources />' }
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'when there is a <sources>' do
context 'and has a single source with a pattern for Go projects' do
let(:project_path) { 'local/go' } # Make sure we're not making false positives
let(:sources_xml) do
<<~EOF
<sources>
<source>/usr/local/go/src</source>
</sources>
EOF
end
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'and has multiple sources with a pattern for Go projects' do
let(:project_path) { 'local/go' } # Make sure we're not making false positives
let(:sources_xml) do
<<~EOF
<sources>
<source>/usr/local/go/src</source>
<source>/go/src</source>
</sources>
EOF
end
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'and has a single source but already is at the project root path' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/#{project_path}</source>
</sources>
EOF
end
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'and has multiple sources but already are at the project root path' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/#{project_path}/</source>
<source>builds/somewhere/#{project_path}</source>
</sources>
EOF
end
it_behaves_like 'ignoring sources, project_path, and worktree_paths'
end
context 'and has a single source that is not at the project root path' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/#{project_path}/app</source>
</sources>
EOF
end
context 'when there is no <class>' do
let(:classes_xml) { '' }
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'when there is a single <class>' do
context 'with no lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'with a single line but the filename cannot be determined based on extracted source and worktree paths' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="member.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'with a single line' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2 } })
end
end
context 'with multiple lines and methods info' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0 } })
end
end
end
context 'when there are multiple <class>' do
context 'with the same filename but the filename cannot be determined based on extracted source and worktree paths' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="member.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="member.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'without a parent package' do
let(:classes_xml) do
<<~EOF
<packages>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="user.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</packages>
EOF
end
it 'parses XML and returns coverage information with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0, 6 => 1, 7 => 1 } })
end
end
context 'with the same filename and different lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="user.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with merged coverage, and with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0, 6 => 1, 7 => 1 } })
end
end
context 'with the same filename and lines' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="1"/>
<line number="2" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with summed-up coverage, and with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 3, 2 => 1 } })
end
end
context 'with missing filename' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and ignores class with missing name' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0 } })
end
end
context 'with filename that cannot be determined based on extracted source and worktree paths' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="member.rb"><methods/><lines>
<line number="6" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and ignores class with undetermined filename' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app/user.rb' => { 1 => 2, 2 => 0 } })
end
end
context 'with invalid line information' do
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><methods/><lines>
<line number="1" hits="2"/>
<line number="2" hits="0"/>
</lines></class>
<class filename="user.rb"><methods/><lines>
<line null="test" hits="1"/>
<line number="7" hits="1"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'raises an error' do
expect { parse_report }.to raise_error(Gitlab::Ci::Parsers::Coverage::Cobertura::InvalidLineInformationError)
end
end
end
end
context 'and has multiple sources that are not at the project root path' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/#{project_path}/app1/</source>
<source>builds/#{project_path}/app2/</source>
</sources>
EOF
end
context 'and a class filename is available under multiple extracted sources' do
let(:paths) { ['app1/user.rb', 'app2/user.rb'] }
let(:classes_xml) do
<<~EOF
<package name="app1">
<classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes>
</package>
<package name="app2">
<classes>
<class filename="user.rb"><lines>
<line number="2" hits="3"/>
</lines></class>
</classes>
</package>
EOF
end
it 'parses XML and returns the files with the filename relative to project root' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({
'app1/user.rb' => { 1 => 2 },
'app2/user.rb' => { 2 => 3 }
})
end
end
context 'and a class filename is available under one of the extracted sources' do
let(:paths) { ['app1/member.rb', 'app2/user.rb', 'app2/pet.rb'] }
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns a single file with the filename relative to project root using the extracted source where it is first found under' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'app2/user.rb' => { 1 => 2 } })
end
end
context 'and a class filename is not found under any of the extracted sources' do
let(:paths) { ['app1/member.rb', 'app2/pet.rb'] }
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
context 'and a class filename is not found under any of the extracted sources within the iteratable limit' do
let(:paths) { ['app2/user.rb'] }
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="record.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
before do
stub_const("#{described_class}::MAX_SOURCES", 1)
end
it 'parses XML and returns empty coverage' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({})
end
end
end
end
shared_examples_for 'non-smart parsing' do
let(:sources_xml) do
<<~EOF
<sources>
<source>builds/foo/bar/app</source>
</sources>
EOF
end
let(:classes_xml) do
<<~EOF
<packages><package name="app"><classes>
<class filename="user.rb"><lines>
<line number="1" hits="2"/>
</lines></class>
</classes></package></packages>
EOF
end
it 'parses XML and returns filenames unchanged just as how they are found in the class node' do
expect { parse_report }.not_to raise_error
expect(coverage_report.files).to eq({ 'user.rb' => { 1 => 2 } })
end
end
context 'when project_path is not present' do
let(:project_path) { nil }
let(:paths) { ['app/user.rb'] }
it_behaves_like 'non-smart parsing'
end
context 'when worktree_paths is not present' do
let(:project_path) { 'foo/bar' }
let(:paths) { nil }
it_behaves_like 'non-smart parsing'
end
end
context 'when data is not Cobertura style XML' do
let(:cobertura) { { coverage: '12%' }.to_json }
it 'raises an error' do
expect { parse_report }.to raise_error(Gitlab::Ci::Parsers::Coverage::Cobertura::InvalidXMLError)
end
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7