Commit 9e4d74e9 authored by Dmytro Zaporozhets (DZ)'s avatar Dmytro Zaporozhets (DZ)

Merge branch 'parse-scan-object' into 'master'

Add scan object to fixtures and refactor common parser

See merge request gitlab-org/gitlab!42838
parents 82f7405e d2b5ac90
......@@ -11,13 +11,13 @@ module Gitlab
report_data = parse_report(json_data)
raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash)
report.scanned_resources = create_scanned_resources(report_data.dig('scan', 'scanned_resources'))
create_scanner(report, report_data.dig('scan', 'scanner'))
collate_remediations(report_data).each do |vulnerability|
create_vulnerability(report, vulnerability, report_data["version"])
end
report_data
rescue JSON::ParserError
raise SecurityReportParserError, 'JSON parsing failed'
rescue => e
......@@ -27,17 +27,6 @@ module Gitlab
protected
def create_scanned_resources(scanned_resources)
return [] unless scanned_resources
scanned_resources.map do |sr|
uri = URI.parse(sr['url'])
::Gitlab::Ci::Reports::Security::ScannedResource.new(uri, sr['method'])
rescue URI::InvalidURIError
nil
end.compact
end
def parse_report(json_data)
Gitlab::Json.parse!(json_data)
end
......@@ -64,7 +53,7 @@ module Gitlab
end
def create_vulnerability(report, data, version)
scanner = create_scanner(report, data['scanner'] || mutate_scanner_tool(data['tool']))
scanner = create_scanner(report, data['scanner'])
identifiers = create_identifiers(report, data['identifiers'])
report.add_finding(
::Gitlab::Ci::Reports::Security::Finding.new(
......@@ -110,11 +99,6 @@ module Gitlab
url: identifier['url']))
end
# TODO: this can be removed as of `12.0`
def mutate_scanner_tool(tool)
{ 'id' => tool, 'name' => tool.capitalize } if tool
end
def parse_severity_level(input)
return input if ::Vulnerabilities::Finding::SEVERITY_LEVELS.key?(input)
......
......@@ -5,7 +5,13 @@ module Gitlab
module Parsers
module Security
class Dast < Common
protected
def parse!(json_data, report)
report_data = super
report.scanned_resources = create_scanned_resources(report_data.dig('scan', 'scanned_resources'))
end
private
def parse_report(json_data)
report = super
......@@ -15,7 +21,16 @@ module Gitlab
report
end
private
def create_scanned_resources(scanned_resources)
return [] unless scanned_resources
scanned_resources.map do |sr|
uri = URI.parse(sr['url'])
::Gitlab::Ci::Reports::Security::ScannedResource.new(uri, sr['method'])
rescue URI::InvalidURIError
nil
end.compact
end
def create_location(location_data)
::Gitlab::Ci::Reports::Security::Locations::Dast.new(
......
......@@ -301,6 +301,16 @@ FactoryBot.define do
end
end
trait :common_security_report do
file_format { :raw }
file_type { :dependency_scanning }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/master/gl-common-scanning-report.json'), 'application/json')
end
end
trait :container_scanning_feature_branch do
file_format { :raw }
file_type { :container_scanning }
......
......@@ -39,5 +39,18 @@
]
}
],
"remediations": []
}
"remediations": [],
"scan": {
"scanner": {
"id": "clair",
"name": "Clair",
"url": "https://github.com/coreos/clair",
"vendor": {
"name": "GitLab"
},
"version": "2.1.4"
},
"type": "container_scanning",
"status": "success"
}
}
\ No newline at end of file
......@@ -166,5 +166,20 @@
"url": "https://github.com/ffi/ffi/releases/tag/1.9.24",
"tool": "bundler_audit"
}
]
],
"scan": {
"scanner": {
"id": "gemnasium-maven",
"name": "gemnasium-maven",
"url": "https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven",
"vendor": {
"name": "GitLab"
},
"version": "2.18.0"
},
"type": "dependency_scanning",
"start_time": "placeholder-value",
"end_time": "placeholder-value",
"status": "success"
}
}
......@@ -857,5 +857,21 @@
"url": "https://cwe.mitre.org/data/definitions/119.html",
"tool": "flawfinder"
}
]
],
"remediations": [],
"scan": {
"scanner": {
"id": "gosec",
"name": "Gosec",
"url": "https://github.com/securego/gosec",
"vendor": {
"name": "GitLab"
},
"version": "2.3.0"
},
"type": "sast",
"status": "success",
"start_time": "placeholder-value",
"end_time": "placeholder-value"
}
}
{
"vulnerabilities": [
{
"category": "dependency_scanning",
"name": "Vulnerabilities in libxml2",
"message": "Vulnerabilities in libxml2 in nokogiri",
"description": "",
"cve": "CVE-1020",
"severity": "High",
"solution": "Upgrade to latest version.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [
{
"url": ""
}
]
},
{
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3",
"category": "dependency_scanning",
"name": "Regular Expression Denial of Service",
"message": "Regular Expression Denial of Service in debug",
"description": "",
"cve": "CVE-1030",
"severity": "Unknown",
"solution": "Upgrade to latest versions.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [
{
"url": ""
}
]
},
{
"category": "dependency_scanning",
"name": "Authentication bypass via incorrect DOM traversal and canonicalization",
"message": "Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js",
"description": "",
"cve": "yarn/yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98",
"severity": "Unknown",
"solution": "Upgrade to fixed version.\r\n",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [
{
"url": ""
},
{
"url": ""
}
]
}
],
"remediations": [
{
"fixes": [
{
"cve": "CVE-1020"
}
],
"summary": "",
"diff": ""
},
{
"fixes": [
{
"cve": "CVE",
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3"
}
],
"summary": "",
"diff": ""
},
{
"fixes": [
{
"cve": "CVE",
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3"
}
],
"summary": "",
"diff": ""
},
{
"fixes": [
{
"id": "2134",
"cve": "CVE-1"
}
],
"summary": "",
"diff": ""
}
],
"dependency_files": [],
"scan": {
"scanner": {
"id": "gemnasium",
"name": "Gemnasium",
"url": "https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven",
"vendor": {
"name": "GitLab"
},
"version": "2.18.0"
},
"type": "dependency_scanning",
"start_time": "placeholder-value",
"end_time": "placeholder-value",
"status": "success"
}
}
\ No newline at end of file
......@@ -285,5 +285,18 @@
]
}
],
"remediations": []
}
"remediations": [],
"scan": {
"scanner": {
"id": "clair",
"name": "Clair",
"url": "https://github.com/coreos/clair",
"vendor": {
"name": "GitLab"
},
"version": "2.1.4"
},
"type": "container_scanning",
"status": "success"
}
}
\ No newline at end of file
......@@ -177,5 +177,21 @@
"url": "https://github.com/ffi/ffi/releases/tag/1.9.24",
"tool": "bundler_audit"
}
]
],
"scan": {
"scanner": {
"id": "gemnasium-maven",
"name": "gemnasium-maven",
"url": "https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven",
"vendor": {
"name": "GitLab"
},
"version": "2.18.0"
},
"type": "dependency_scanning",
"start_time": "placeholder-value",
"end_time": "placeholder-value",
"status": "success"
}
}
......@@ -963,5 +963,21 @@
"url": "https://cwe.mitre.org/data/definitions/120.html",
"tool": "flawfinder"
}
]
],
"remediations": [],
"scan": {
"scanner": {
"id": "gosec",
"name": "Gosec",
"url": "https://github.com/securego/gosec",
"vendor": {
"name": "GitLab"
},
"version": "2.3.0"
},
"type": "sast",
"status": "success",
"start_time": "placeholder-value",
"end_time": "placeholder-value"
}
}
......@@ -6,236 +6,32 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
describe '#parse!' do
let_it_be(:pipeline) { create(:ci_pipeline) }
let(:artifact) { build(:ee_ci_job_artifact, :dependency_scanning) }
let(:artifact) { build(:ee_ci_job_artifact, :common_security_report) }
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, 2.weeks.ago) }
let(:parser) { described_class.new }
before do
allow(parser).to receive(:create_location).and_return(nil)
end
it 'converts undefined severity and confidence' do
artifact.each_blob do |blob|
blob.gsub!("Unknown", "Undefined")
parser.parse!(blob, report)
end
expect(report.findings.map(&:severity)).to include("unknown")
expect(report.findings.map(&:confidence)).to include("unknown")
expect(report.findings.map(&:severity)).not_to include("undefined")
expect(report.findings.map(&:confidence)).not_to include("undefined")
end
context 'parsing scanned resources' do
describe 'when the URL is invalid' do
let(:raw_json) do
{
"vulnerabilities": [],
"remediations": [],
"dependency_files": [],
"scan": {
"scanned_resources": [
{
"method": "GET",
"type": "url",
"url": "not a URL"
}
]
}
}
end
it 'skips invalid URLs' do
parser.parse!(raw_json.to_json, report)
expect(report.scanned_resources).to be_empty
end
end
describe 'when the URLs are valid' do
let(:raw_json) do
{
"vulnerabilities": [],
"remediations": [],
"dependency_files": [],
"scan": {
"scanned_resources": [
{
"method": "GET",
"type": "url",
"url": "http://example.com/1"
},
{
"method": "POST",
"type": "url",
"url": "http://example.com/2"
},
{
"method": "GET",
"type": "url",
"url": "http://example.com/3"
}
]
}
}
end
it 'creates a scanned resource for each URL' do
parser.parse!(raw_json.to_json, report)
expect(report.scanned_resources.length).to eq(3)
end
it 'converts the JSON to Scanned Resource objects' do
parser.parse!(raw_json.to_json, report)
expect(report.scanned_resources.first).to be_a(::Gitlab::Ci::Reports::Security::ScannedResource)
end
end
end
context 'parsing remediations' do
let(:raw_json) do
{
"vulnerabilities": [
{
"category": "dependency_scanning",
"name": "Vulnerabilities in libxml2",
"message": "Vulnerabilities in libxml2 in nokogiri",
"description": "",
"cve": "CVE-1020",
"severity": "High",
"solution": "Upgrade to latest version.",
"scanner": { "id": "gemnasium", "name": "Gemnasium" },
"location": {},
"identifiers": [],
"links": [{ "url": "" }]
},
{
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3",
"category": "dependency_scanning",
"name": "Regular Expression Denial of Service",
"message": "Regular Expression Denial of Service in debug",
"description": "",
"cve": "CVE-1030",
"severity": "Unknown",
"solution": "Upgrade to latest versions.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [{ "url": "" }]
},
{
"category": "dependency_scanning",
"name": "Authentication bypass via incorrect DOM traversal and canonicalization",
"message": "Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js",
"description": "",
"cve": "yarn/yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98",
"severity": "Unknown",
"solution": "Upgrade to fixed version.\r\n",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [{ "url": "" }, { "url": "" }]
}
],
"remediations": [],
"dependency_files": [],
"scan": {
"scanner": {
"id": "gemnasium",
"name": "Gemnasium",
"vendor": { "name": "GitLab" }
}
}
}
end
it 'finds remediation with same cve' do
fix_with_cve = {
"fixes": [
{
"cve": "CVE-1020"
}
],
"summary": "",
"diff": ""
}
raw_json[:remediations] << fix_with_cve
parser.parse!(raw_json.to_json, report)
vulnerability = report.findings.find { |x| x.compare_key == "CVE-1020" }
expect(vulnerability.raw_metadata).to include fix_with_cve.to_json
remediation = { 'fixes' => [{ 'cve' => 'CVE-1020' }], 'summary' => '', 'diff' => '' }
expect(Gitlab::Json.parse(vulnerability.raw_metadata).dig('remediations').first).to include remediation
end
it 'finds remediation with same id' do
fix_with_id = {
"fixes": [
{
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3",
"cve": "CVE"
}
],
"summary": "",
"diff": ""
}
raw_json[:remediations] << fix_with_id
parser.parse!(raw_json.to_json, report)
vulnerability = report.findings.find { |x| x.compare_key == "CVE-1030" }
expect(vulnerability.raw_metadata).to include fix_with_id.to_json
end
it 'finds cve and id' do
fix_with_id = {
"fixes": [
{
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3",
"cve": "CVE"
}
],
"summary": "",
"diff": ""
}
fix_with_cve = {
"fixes": [
{
"cve": "CVE-1020"
}
],
"summary": "",
"diff": ""
}
raw_json[:remediations] << fix_with_id << fix_with_cve
parser.parse!(raw_json.to_json, report)
vulnerability_1030 = report.findings.find { |x| x.compare_key == "CVE-1030" }
expect(vulnerability_1030.raw_metadata).to include fix_with_id.to_json
vulnerability_1020 = report.findings.find { |x| x.compare_key == "CVE-1020" }
expect(vulnerability_1020.raw_metadata).to include fix_with_cve.to_json
remediation = { 'fixes' => [{ 'cve' => 'CVE', 'id' => 'bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3' }], 'summary' => '', 'diff' => '' }
expect(Gitlab::Json.parse(vulnerability.raw_metadata).dig('remediations').first).to include remediation
end
it 'does not find remediation with different id' do
fix_with_id = {
"fixes": [
{
"id": "bb2f",
"cve": "CVE"
}
],
"summary": "",
"diff": ""
}
fix_with_id_2 = {
"fixes": [
{
"id": "2134",
......@@ -246,55 +42,16 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
"diff": ""
}
raw_json[:remediations] << fix_with_id << fix_with_id_2
parser.parse!(raw_json.to_json, report)
report.findings.map do |vulnerability|
expect(vulnerability.raw_metadata).not_to include(fix_with_id.to_json)
expect(Gitlab::Json.parse(vulnerability.raw_metadata).dig('remediations')).not_to include(fix_with_id)
end
end
end
context 'parsing scanners' do
let(:raw_json) do
{
"vulnerabilities": [
{
"category": "dependency_scanning",
"name": "Vulnerabilities in libxml2",
"message": "Vulnerabilities in libxml2 in nokogiri",
"description": "",
"cve": "CVE-1020",
"severity": "High",
"solution": "Upgrade to latest version.",
"scanner": raw_scanner,
"location": {},
"identifiers": [],
"links": [{ "url": "" }]
}
],
"remediations": [],
"dependency_files": []
}
end
subject(:scanner) { report.findings.first.scanner }
before do
parser.parse!(raw_json.to_json, report)
end
context 'when vendor is missing in scanner' do
let(:raw_scanner) { { 'id': 'gemnasium', 'name': 'Gemnasium' } }
it 'returns scanner with empty vendor field' do
expect(scanner.vendor).to be_nil
end
end
context 'when vendor is not missing in scanner' do
let(:raw_scanner) { { 'id': 'gemnasium', 'name': 'Gemnasium', 'vendor': { 'name': 'GitLab' } } }
it 'returns scanner with parsed vendor value' do
expect(scanner.vendor).to eq('GitLab')
end
......
......@@ -21,7 +21,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::ContainerScanning do
it "parses all identifiers and findings for unapproved vulnerabilities" do
expect(report.findings.length).to eq(8)
expect(report.identifiers.length).to eq(8)
expect(report.scanners.length).to eq(1)
expect(report.scanners.length).to eq(2)
end
it 'generates expected location' do
......
......@@ -46,7 +46,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Dast do
it 'generates expected location' do
location = report.findings.last.location
expect(location).to be_a(::Gitlab::Ci::Reports::Security::Locations::Dast)
expect(location).to have_attributes(
hostname: last_occurrence_hostname,
......@@ -71,5 +70,42 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Dast do
end
end
end
describe 'parses scanned_resources' do
let(:artifact) { create(:ee_ci_job_artifact, 'dast') }
before do
artifact.each_blob do |blob|
parser.parse!(blob, report)
end
end
let(:raw_json) do
{
"vulnerabilities": [],
"remediations": [],
"dependency_files": [],
"scan": {
"scanned_resources": [
{
"method": "GET",
"type": "url",
"url": "not a URL"
}
]
}
}
end
it 'skips invalid URLs' do
parser.parse!(raw_json.to_json, report)
expect(report.scanned_resources).to be_empty
end
it 'creates a scanned resource for each URL' do
expect(report.scanned_resources.length).to eq(6)
expect(report.scanned_resources.first).to be_a(::Gitlab::Ci::Reports::Security::ScannedResource)
end
end
end
end
......@@ -13,7 +13,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::DependencyScanning do
let(:parser) { described_class.new }
where(:report_format, :occurrence_count, :identifier_count, :scanner_count, :file_path, :package_name, :package_version, :version) do
:dependency_scanning | 4 | 7 | 2 | 'app/pom.xml' | 'io.netty/netty' | '3.9.1.Final' | '1.3'
:dependency_scanning | 4 | 7 | 3 | 'app/pom.xml' | 'io.netty/netty' | '3.9.1.Final' | '1.3'
:dependency_scanning_deprecated | 4 | 7 | 2 | 'app/pom.xml' | 'io.netty/netty' | '3.9.1.Final' | '1.3'
:dependency_scanning_remediation | 2 | 3 | 1 | 'yarn.lock' | 'debug' | '1.0.5' | '2.0'
end
......
......@@ -3,6 +3,8 @@
require 'spec_helper'
RSpec.describe Gitlab::Ci::Parsers::Security::Sast do
using RSpec::Parameterized::TableSyntax
describe '#parse!' do
let_it_be(:pipeline) { create(:ci_pipeline) }
......@@ -11,7 +13,10 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do
subject(:parser) { described_class.new }
context "when parsing valid reports" do
where(report_format: %i(sast sast_deprecated))
where(:report_format, :scanner_length) do
:sast | 4
:sast_deprecated | 3
end
with_them do
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, created_at) }
......@@ -26,7 +31,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do
it "parses all identifiers and findings" do
expect(report.findings.length).to eq(33)
expect(report.identifiers.length).to eq(17)
expect(report.scanners.length).to eq(3)
expect(report.scanners.length).to eq(scanner_length)
end
it 'generates expected location' do
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment