Skip to content

G
gitlab-ce
Commit 19ae43b3 authored by Adam Cohen's avatar Adam Cohen Committed by charlie ablett
Browse files
Options

Remove deprecated container scanning report parser

parent a91546fe
Show whitespace changes
Inline Side-by-side
Showing with 284 additions and 720 deletions
changelogs/unreleased/remove-deprecated-container-scanning-report-format.yml 0 → 100644
View file @ 19ae43b3
---
title: Remove deprecated container scanning report parser
merge_request: 31294
author:
type: removed
ee/lib/gitlab/ci/parsers/security/container_scanning.rb
View file @ 19ae43b3
...@@ -5,41 +5,8 @@ module Gitlab ...@@ -5,41 +5,8 @@ module Gitlab
module Parsers module Parsers
module Security module Security
class ContainerScanning < Common class ContainerScanning < Common
include Security::Concerns::DeprecatedSyntax
DEPRECATED_REPORT_VERSION = "1.3".freeze
def parse_report(json_data)
report = super
return format_deprecated_report(report) if deprecated?(report)
report
end
private private
# Transforms the clair-scanner JSON report into the expected format
# TODO: remove the following block when we no longer need to support legacy
# clair-scanner data. See https://gitlab.com/gitlab-org/gitlab/issues/35442
def format_deprecated_report(data)
unapproved = data['unapproved']
formatter = Formatters::DeprecatedContainerScanning.new(data['image'])
vulnerabilities = data['vulnerabilities'].map do |vulnerability|
# We only report unapproved vulnerabilities
next unless unapproved.include?(vulnerability['vulnerability'])
formatter.format(vulnerability)
end.compact
{ "vulnerabilities" => vulnerabilities, "version" => DEPRECATED_REPORT_VERSION }
end
def deprecated?(data)
data['image']
end
def create_location(location_data) def create_location(location_data)
::Gitlab::Ci::Reports::Security::Locations::ContainerScanning.new( ::Gitlab::Ci::Reports::Security::Locations::ContainerScanning.new(
image: location_data['image'], image: location_data['image'],
......
ee/lib/gitlab/ci/parsers/security/formatters/deprecated_container_scanning.rb deleted 100644 → 0
View file @ a91546fe
# frozen_string_literal: true
# TODO: remove this class when we no longer need to support legacy
# clair-scanner data. See https://gitlab.com/gitlab-org/gitlab/issues/35442
module Gitlab
module Ci
module Parsers
module Security
module Formatters
class DeprecatedContainerScanning
def initialize(image)
@image = image
end
def format(vulnerability)
formatted_vulnerability = DeprecatedFormattedContainerScanningVulnerability.new(vulnerability)
{
'category' => 'container_scanning',
'message' => formatted_vulnerability.message,
'description' => formatted_vulnerability.description,
'cve' => formatted_vulnerability.cve,
'severity' => formatted_vulnerability.severity,
'solution' => formatted_vulnerability.solution,
'confidence' => 'Unknown',
'location' => {
'image' => image,
'operating_system' => formatted_vulnerability.operating_system,
'dependency' => {
'package' => {
'name' => formatted_vulnerability.package_name
},
'version' => formatted_vulnerability.version
}
},
'scanner' => { 'id' => 'clair', 'name' => 'Clair' },
'identifiers' => [
{
'type' => 'cve',
'name' => formatted_vulnerability.cve,
'value' => formatted_vulnerability.cve,
'url' => formatted_vulnerability.url
}
],
'links' => [{ 'url' => formatted_vulnerability.url }]
}
end
private
attr_reader :image
end
end
end
end
end
end
ee/lib/gitlab/ci/parsers/security/formatters/deprecated_formatted_container_scanning_vulnerability.rb deleted 100644 → 0
View file @ a91546fe
# frozen_string_literal: true
# TODO: remove this class when we no longer need to support legacy
# clair-scanner data. See https://gitlab.com/gitlab-org/gitlab/issues/35442
module Gitlab
module Ci
module Parsers
module Security
module Formatters
class DeprecatedFormattedContainerScanningVulnerability
def initialize(vulnerability)
@vulnerability = vulnerability
end
def message
@message ||= format_definitions(
%w[vulnerability featurename] => '%{vulnerability} in %{featurename}',
'vulnerability' => '%{vulnerability}'
)
end
def description
@description ||= format_definitions(
'description' => '%{description}',
%w[featurename featureversion] => '%{featurename}:%{featureversion} is affected by %{vulnerability}',
'featurename' => '%{featurename} is affected by %{vulnerability}',
'namespace' => '%{namespace} is affected by %{vulnerability}'
)
end
def severity
raw_severity = vulnerability['severity']
@severity ||= case raw_severity
when 'Negligible'
'low'
when 'Unknown', 'Low', 'Medium', 'High', 'Critical'
raw_severity.downcase
when 'Defcon1'
'critical'
else
safe_severity = ERB::Util.html_escape(raw_severity)
raise(
::Gitlab::Ci::Parsers::Security::Common::SecurityReportParserError,
"Unknown severity in container scanning report: #{safe_severity}"
)
end
end
def solution
@solution ||= format_definitions(
%w[fixedby featurename featureversion] => 'Upgrade %{featurename} from %{featureversion} to %{fixedby}',
%w[fixedby featurename] => 'Upgrade %{featurename} to %{fixedby}',
'fixedby' => 'Upgrade to %{fixedby}'
)
end
def cve
@cve ||= vulnerability['vulnerability']
end
def operating_system
@operating_system ||= vulnerability['namespace']
end
def package_name
@package_name ||= vulnerability['featurename']
end
def version
@version ||= vulnerability['featureversion']
end
def url
@url ||= vulnerability['link']
end
private
attr_reader :vulnerability
def format_definitions(definitions)
find_definitions(definitions).then do |_, value|
if value.present?
value % vulnerability.symbolize_keys
end
end
end
def find_definitions(definitions)
definitions.find do |keys, value|
vulnerability.values_at(*keys).all?(&:present?)
end
end
end
end
end
end
end
end
ee/spec/factories/ci/builds.rb
View file @ 19ae43b3
...@@ -72,12 +72,6 @@ FactoryBot.define do ...@@ -72,12 +72,6 @@ FactoryBot.define do
end end
end end
trait :deprecated_container_scanning_report do
after(:build) do |build|
build.job_artifacts << create(:ee_ci_job_artifact, :deprecated_container_scanning_report, job: build)
end
end
trait :dependency_scanning_feature_branch do trait :dependency_scanning_feature_branch do
after(:build) do |build| after(:build) do |build|
build.job_artifacts << create(:ee_ci_job_artifact, :dependency_scanning_feature_branch, job: build) build.job_artifacts << create(:ee_ci_job_artifact, :dependency_scanning_feature_branch, job: build)
......
ee/spec/factories/ci/job_artifacts.rb
View file @ 19ae43b3
...@@ -259,16 +259,6 @@ FactoryBot.define do ...@@ -259,16 +259,6 @@ FactoryBot.define do
end end
end end
trait :deprecated_container_scanning_report do
file_format { :raw }
file_type { :container_scanning }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/deprecated/gl-container-scanning-report.json'), 'text/plain')
end
end
trait :metrics do trait :metrics do
file_format { :gzip } file_format { :gzip }
file_type { :metrics } file_type { :metrics }
......
ee/spec/finders/security/pipeline_vulnerabilities_finder_spec.rb
View file @ 19ae43b3
...@@ -11,20 +11,21 @@ describe Security::PipelineVulnerabilitiesFinder do ...@@ -11,20 +11,21 @@ describe Security::PipelineVulnerabilitiesFinder do
let_it_be(:project) { create(:project, :repository) } let_it_be(:project) { create(:project, :repository) }
let_it_be(:pipeline) { create(:ci_pipeline, :success, project: project) } let_it_be(:pipeline) { create(:ci_pipeline, :success, project: project) }
let_it_be(:build_cs) { create(:ci_build, :success, name: 'cs_job', pipeline: pipeline, project: project) }
shared_examples_for 'a pipeline vulnerabilities finder' do
describe '#execute' do describe '#execute' do
let(:params) { {} } let(:params) { {} }
let_it_be(:build_cs) { create(:ci_build, :success, name: 'cs_job', pipeline: pipeline, project: project) }
let_it_be(:build_dast) { create(:ci_build, :success, name: 'dast_job', pipeline: pipeline, project: project) } let_it_be(:build_dast) { create(:ci_build, :success, name: 'dast_job', pipeline: pipeline, project: project) }
let_it_be(:build_ds) { create(:ci_build, :success, name: 'ds_job', pipeline: pipeline, project: project) } let_it_be(:build_ds) { create(:ci_build, :success, name: 'ds_job', pipeline: pipeline, project: project) }
let_it_be(:build_sast) { create(:ci_build, :success, name: 'sast_job', pipeline: pipeline, project: project) } let_it_be(:build_sast) { create(:ci_build, :success, name: 'sast_job', pipeline: pipeline, project: project) }
let_it_be(:artifact_cs) { create(:ee_ci_job_artifact, :container_scanning, job: build_cs, project: project) }
let_it_be(:artifact_dast) { create(:ee_ci_job_artifact, :dast, job: build_dast, project: project) } let_it_be(:artifact_dast) { create(:ee_ci_job_artifact, :dast, job: build_dast, project: project) }
let_it_be(:artifact_ds) { create(:ee_ci_job_artifact, :dependency_scanning, job: build_ds, project: project) } let_it_be(:artifact_ds) { create(:ee_ci_job_artifact, :dependency_scanning, job: build_ds, project: project) }
let_it_be(:artifact_sast) { create(:ee_ci_job_artifact, :sast, job: build_sast, project: project) } let_it_be(:artifact_sast) { create(:ee_ci_job_artifact, :sast, job: build_sast, project: project) }
let(:cs_count) { read_fixture(artifact_cs)['vulnerabilities'].count }
let(:ds_count) { read_fixture(artifact_ds)['vulnerabilities'].count } let(:ds_count) { read_fixture(artifact_ds)['vulnerabilities'].count }
let(:sast_count) { read_fixture(artifact_sast)['vulnerabilities'].count } let(:sast_count) { read_fixture(artifact_sast)['vulnerabilities'].count }
let(:dast_count) do let(:dast_count) do
...@@ -302,19 +303,4 @@ describe Security::PipelineVulnerabilitiesFinder do ...@@ -302,19 +303,4 @@ describe Security::PipelineVulnerabilitiesFinder do
Gitlab::Json.parse(File.read(fixture.file.path)) Gitlab::Json.parse(File.read(fixture.file.path))
end end
end end
end
context 'container_scanning' do
let_it_be(:artifact_cs) { create(:ee_ci_job_artifact, :container_scanning, job: build_cs, project: project) }
let(:cs_count) { read_fixture(artifact_cs)['vulnerabilities'].count }
it_behaves_like 'a pipeline vulnerabilities finder'
end
context 'deprecated container_scanning' do
let_it_be(:artifact_cs) { create(:ee_ci_job_artifact, :deprecated_container_scanning_report, job: build_cs, project: project) }
let(:cs_count) { read_fixture(artifact_cs)['unapproved'].count }
it_behaves_like 'a pipeline vulnerabilities finder'
end
end end
ee/spec/fixtures/security_reports/feature-branch/gl-container-scanning-report.json
View file @ 19ae43b3
{ {
"image": "registry.gitlab.com/bikebilly/auto-devops-10-6/feature-branch:e7315ba964febb11bac8f5cd6ec433db8a3a1583", "version": "2.4",
"unapproved": ["CVE-2017-15650"],
"vulnerabilities": [ "vulnerabilities": [
{ {
"featurename": "musl", "id": "e987fa54ff94e1d0e716814861459d2eb10bd27a0ba8ca243428669d8885ce68",
"featureversion": "1.1.14-r15", "category": "container_scanning",
"vulnerability": "CVE-2017-15650", "message": "CVE-2017-15650 in musl",
"namespace": "alpine:v3.4", "description": "musl:1.1.18-r3 is affected by CVE-2017-15650",
"description": "", "cve": "alpine:v3.7:musl:CVE-2017-15650",
"link": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15650", "severity": "High",
"severity": "Medium", "confidence": "Unknown",
"fixedby": "1.1.14-r16" "solution": "Upgrade musl from 1.1.18-r3 to 1.1.18-r4",
"scanner": {
"id": "klar",
"name": "klar"
},
"location": {
"dependency": {
"package": {
"name": "musl"
},
"version": "1.1.18-r3"
},
"operating_system": "alpine:v3.7",
"image": "registry.gitlab.com/bikebilly/auto-devops-10-6/feature-branch:e7315ba964febb11bac8f5cd6ec433db8a3a1583"
},
"identifiers": [
{
"type": "cve",
"name": "CVE-2017-15650",
"value": "CVE-2017-15650",
"url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15650"
}
],
"links": [
{
"url": "https://cve.mitre.org/cgi-bin/cvename.cgi?name=CVE-2017-15650"
} }
] ]
}
],
"remediations": []
} }
ee/spec/lib/gitlab/ci/parsers/security/container_scanning_spec.rb
View file @ 19ae43b3
...@@ -15,15 +15,8 @@ describe Gitlab::Ci::Parsers::Security::ContainerScanning do ...@@ -15,15 +15,8 @@ describe Gitlab::Ci::Parsers::Security::ContainerScanning do
end end
describe '#parse!' do describe '#parse!' do
using RSpec::Parameterized::TableSyntax let(:artifact) { create(:ee_ci_job_artifact, :container_scanning) }
let(:image) { 'registry.gitlab.com/gitlab-org/security-products/dast/webgoat-8.0@sha256:bc09fe2e0721dfaeee79364115aeedf2174cce0947b9ae5fe7c33312ee019a4e' }
where(:report_type, :image, :version) do
:deprecated_container_scanning_report | 'registry.gitlab.com/groulot/container-scanning-test/master:5f21de6956aee99ddb68ae49498662d9872f50ff' | '1.3'
:container_scanning | 'registry.gitlab.com/gitlab-org/security-products/dast/webgoat-8.0@sha256:bc09fe2e0721dfaeee79364115aeedf2174cce0947b9ae5fe7c33312ee019a4e' | '2.3'
end
with_them do
let(:artifact) { create(:ee_ci_job_artifact, report_type) }
it "parses all identifiers and occurrences for unapproved vulnerabilities" do it "parses all identifiers and occurrences for unapproved vulnerabilities" do
expect(report.occurrences.length).to eq(8) expect(report.occurrences.length).to eq(8)
...@@ -44,12 +37,11 @@ describe Gitlab::Ci::Parsers::Security::ContainerScanning do ...@@ -44,12 +37,11 @@ describe Gitlab::Ci::Parsers::Security::ContainerScanning do
end end
it "generates expected metadata_version" do it "generates expected metadata_version" do
expect(report.occurrences.first.metadata_version).to eq(version) expect(report.occurrences.first.metadata_version).to eq('2.3')
end end
it "adds report image's name to raw_metadata" do it "adds report image's name to raw_metadata" do
expect(Gitlab::Json.parse(report.occurrences.first.raw_metadata).dig('location', 'image')).to eq(image) expect(Gitlab::Json.parse(report.occurrences.first.raw_metadata).dig('location', 'image')).to eq(image)
end end
end end
end
end end
ee/spec/lib/gitlab/ci/parsers/security/formatters/deprecated_container_scanning_spec.rb deleted 100644 → 0
View file @ a91546fe
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Parsers::Security::Formatters::DeprecatedContainerScanning do
let(:vulnerability) { raw_report['vulnerabilities'].first }
describe '#format' do
let(:raw_report) do
Gitlab::Json.parse!(
File.read(
Rails.root.join('ee/spec/fixtures/security_reports/deprecated/gl-container-scanning-report.json')
)
)
end
it 'formats the vulnerability into the 1.3 format' do
formatter = described_class.new('image_name')
expect(formatter.format(vulnerability)).to eq( {
'category' => 'container_scanning',
'message' => 'CVE-2017-18269 in glibc',
'confidence' => 'Unknown',
'cve' => 'CVE-2017-18269',
'identifiers' => [
{
'type' => 'cve',
'name' => 'CVE-2017-18269',
'value' => 'CVE-2017-18269',
'url' => 'https://security-tracker.debian.org/tracker/CVE-2017-18269'
}
],
'location' => {
'image' => 'image_name',
'operating_system' => 'debian:9',
'dependency' => {
'package' => {
'name' => 'glibc'
},
'version' => '2.24-11+deb9u3'
}
},
'links' => [{ 'url' => 'https://security-tracker.debian.org/tracker/CVE-2017-18269' }],
'description' => 'SSE2-optimized memmove implementation problem.',
'scanner' => { 'id' => 'clair', 'name' => 'Clair' },
'severity' => 'critical',
'solution' => 'Upgrade glibc from 2.24-11+deb9u3 to 2.24-11+deb9u4'
} )
end
end
end
ee/spec/lib/gitlab/ci/parsers/security/formatters/deprecated_formatted_container_scanning_vulnerability_spec.rb deleted 100644 → 0
View file @ a91546fe
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::Ci::Parsers::Security::Formatters::DeprecatedFormattedContainerScanningVulnerability do
let(:raw_report) do
Gitlab::Json.parse!(
File.read(
Rails.root.join('ee/spec/fixtures/security_reports/deprecated/gl-container-scanning-report.json')
)
)
end
let(:vulnerability) { raw_report['vulnerabilities'].first }
let(:data_with_all_keys) do
raw_report['vulnerabilities'].first.merge(
'description' => 'Better hurry and fix that.',
'featurename' => 'hexes',
'featureversion' => '6.6.6',
'fixedby' => '6.6.7',
'link' => 'https://theintercept.com',
'namespace' => 'malevolences',
'vulnerability' => 'Level 9000 Soul Breach'
)
end
subject { described_class.new(data_with_all_keys) }
describe '#message' do
it 'creates a message from the vulnerability and featurename' do
expect(subject.message).to eq('Level 9000 Soul Breach in hexes')
end
context 'when there is no featurename' do
it 'uses vulnerability for the message' do
data_without_featurename = data_with_all_keys.deep_dup.merge('featurename' => '')
formatted_vulnerability = described_class.new(data_without_featurename)
expect(formatted_vulnerability.message).to eq('Level 9000 Soul Breach')
end
end
end
describe '#description' do
it 'uses the given description' do
expect(subject.description).to eq('Better hurry and fix that.')
end
context 'when there is no description' do
let(:data_without_description) { data_with_all_keys.deep_dup.merge('description' => '') }
it 'creates a description from the featurename and featureversion' do
formatted_vulnerability = described_class.new(data_without_description)
expect(formatted_vulnerability.description).to eq('hexes:6.6.6 is affected by Level 9000 Soul Breach')
end
context 'when there is no featureversion' do
it 'creates a description from the featurename' do
data_without_featureversion = data_without_description.deep_dup.merge('featureversion' => '')
formatted_vulnerability = described_class.new(data_without_featureversion)
expect(formatted_vulnerability.description).to eq('hexes is affected by Level 9000 Soul Breach')
end
end
context 'when there is no featurename and no featureversion' do
it 'creates a description from the namespace' do
data_only_namespace = data_without_description.deep_dup.merge(
'featurename' => '',
'featureversion' => ''
)
formatted_vulnerability = described_class.new(data_only_namespace)
expect(formatted_vulnerability.description).to eq('malevolences is affected by Level 9000 Soul Breach')
end
end
end
end
describe '#severity' do
using RSpec::Parameterized::TableSyntax
where(:report_severity_category, :gitlab_severity_category) do
'Unknown' | 'unknown'
'Negligible' | 'low'
'Low' | 'low'
'Medium' | 'medium'
'High' | 'high'
'Critical' | 'critical'
'Defcon1' | 'critical'
end
with_them do
it 'translates the severity into our categorization' do
data_with_severity = data_with_all_keys.deep_dup.merge('severity' => report_severity_category)
formatted_vulnerability = described_class.new(data_with_severity)
expect(formatted_vulnerability.severity).to eq(gitlab_severity_category)
end
end
context 'when the given severity is not valid' do
it 'throws a parser error' do
data_with_invalid_severity = vulnerability.deep_dup.merge(
'severity' => 'cats, curses, and <coffee>'
)
formatted_vulnerability = described_class.new(data_with_invalid_severity)
expect { formatted_vulnerability.severity }.to raise_error(
::Gitlab::Ci::Parsers::Security::Common::SecurityReportParserError,
'Unknown severity in container scanning report: cats, curses, and &lt;coffee&gt;'
)
end
end
end
describe '#solution' do
it 'creates a solution from the featurename, featureversion, and fixedby' do
expect(subject.solution).to eq('Upgrade hexes from 6.6.6 to 6.6.7')
end
context 'when there is no featurename' do
it 'formats the solution using fixedby' do
data_without_featurename = data_with_all_keys.deep_dup.merge('featurename' => '')
formatted_vulnerability = described_class.new(data_without_featurename)
expect(formatted_vulnerability.solution).to eq('Upgrade to 6.6.7')
end
end
context 'when there is no featureversion' do
it 'formats a solution using featurename' do
data_without_featureversion = data_with_all_keys.deep_dup.merge('featureversion' => '')
formatted_vulnerability = described_class.new(data_without_featureversion)
expect(formatted_vulnerability.solution).to eq('Upgrade hexes to 6.6.7')
end
end
context 'when there is no fixedby' do
it 'does not include a solution' do
data_without_fixedby = vulnerability.deep_dup.merge('fixedby' => '')
formatted_vulnerability = described_class.new(data_without_fixedby)
expect(formatted_vulnerability.solution).to be_nil
end
end
end
describe '#cve' do
it 'reads the CVE from the vulnerability' do
expect(subject.cve).to eq('Level 9000 Soul Breach')
end
end
describe '#operating_system' do
it 'reads the operating system from the namespace' do
expect(subject.operating_system).to eq('malevolences')
end
end
describe '#package_name' do
it 'reads the package name from the featurename' do
expect(subject.package_name).to eq('hexes')
end
end
describe '#version' do
it 'reads the version from featureversion' do
expect(subject.version).to eq('6.6.6')
end
end
describe '#url' do
it 'reads the url from the link in the report' do
expect(subject.url).to eq('https://theintercept.com')
end
end
end
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7