Skip to content

G
gitlab-ce
Commit d2b5ac90 authored by can eldem's avatar can eldem
Browse files
Options

Refactor parser seperate dast specific fields 

Add fixture for common fields
Add fixture for scan fields
parent 60a60a32
Hide whitespace changes
Inline Side-by-side
Showing with 304 additions and 283 deletions
ee/lib/gitlab/ci/parsers/security/common.rb
View file @ d2b5ac90
......@@ -11,13 +11,13 @@ module Gitlab
report_data = parse_report(json_data)
raise SecurityReportParserError, "Invalid report format" unless report_data.is_a?(Hash)
report.scanned_resources = create_scanned_resources(report_data.dig('scan', 'scanned_resources'))
create_scanner(report, report_data.dig('scan', 'scanner'))
collate_remediations(report_data).each do |vulnerability|
create_vulnerability(report, vulnerability, report_data["version"])
end
report_data
rescue JSON::ParserError
raise SecurityReportParserError, 'JSON parsing failed'
rescue => e
......@@ -27,17 +27,6 @@ module Gitlab
protected
def create_scanned_resources(scanned_resources)
return [] unless scanned_resources
scanned_resources.map do |sr|
uri = URI.parse(sr['url'])
::Gitlab::Ci::Reports::Security::ScannedResource.new(uri, sr['method'])
rescue URI::InvalidURIError
nil
end.compact
end
def parse_report(json_data)
Gitlab::Json.parse!(json_data)
end
......@@ -64,7 +53,7 @@ module Gitlab
end
def create_vulnerability(report, data, version)
scanner = create_scanner(report, data['scanner'] || mutate_scanner_tool(data['tool']))
scanner = create_scanner(report, data['scanner'])
identifiers = create_identifiers(report, data['identifiers'])
report.add_finding(
::Gitlab::Ci::Reports::Security::Finding.new(
......@@ -110,11 +99,6 @@ module Gitlab
url: identifier['url']))
end
# TODO: this can be removed as of `12.0`
def mutate_scanner_tool(tool)
{ 'id' => tool, 'name' => tool.capitalize } if tool
end
def parse_severity_level(input)
return input if ::Vulnerabilities::Finding::SEVERITY_LEVELS.key?(input)
......
ee/lib/gitlab/ci/parsers/security/dast.rb
View file @ d2b5ac90
......@@ -5,7 +5,13 @@ module Gitlab
module Parsers
module Security
class Dast < Common
protected
def parse!(json_data, report)
report_data = super
report.scanned_resources = create_scanned_resources(report_data.dig('scan', 'scanned_resources'))
end
private
def parse_report(json_data)
report = super
......@@ -15,7 +21,16 @@ module Gitlab
report
end
private
def create_scanned_resources(scanned_resources)
return [] unless scanned_resources
scanned_resources.map do |sr|
uri = URI.parse(sr['url'])
::Gitlab::Ci::Reports::Security::ScannedResource.new(uri, sr['method'])
rescue URI::InvalidURIError
nil
end.compact
end
def create_location(location_data)
::Gitlab::Ci::Reports::Security::Locations::Dast.new(
......
ee/spec/factories/ci/job_artifacts.rb
View file @ d2b5ac90
......@@ -301,6 +301,16 @@ FactoryBot.define do
end
end
trait :common_security_report do
file_format { :raw }
file_type { :dependency_scanning }
after(:build) do |artifact, _|
artifact.file = fixture_file_upload(
Rails.root.join('ee/spec/fixtures/security_reports/master/gl-common-scanning-report.json'), 'application/json')
end
end
trait :container_scanning_feature_branch do
file_format { :raw }
file_type { :container_scanning }
......
ee/spec/fixtures/security_reports/feature-branch/gl-container-scanning-report.json
View file @ d2b5ac90
......@@ -39,5 +39,18 @@
]
}
],
"remediations": []
}
"remediations": [],
"scan": {
"scanner": {
"id": "clair",
"name": "Clair",
"url": "https://github.com/coreos/clair",
"vendor": {
"name": "GitLab"
},
"version": "2.1.4"
},
"type": "container_scanning",
"status": "success"
}
}
\ No newline at end of file
ee/spec/fixtures/security_reports/feature-branch/gl-dependency-scanning-report.json
View file @ d2b5ac90
......@@ -166,5 +166,20 @@
"url": "https://github.com/ffi/ffi/releases/tag/1.9.24",
"tool": "bundler_audit"
}
]
],
"scan": {
"scanner": {
"id": "gemnasium-maven",
"name": "gemnasium-maven",
"url": "https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven",
"vendor": {
"name": "GitLab"
},
"version": "2.18.0"
},
"type": "dependency_scanning",
"start_time": "placeholder-value",
"end_time": "placeholder-value",
"status": "success"
}
}
ee/spec/fixtures/security_reports/feature-branch/gl-sast-report.json
View file @ d2b5ac90
......@@ -857,5 +857,21 @@
"url": "https://cwe.mitre.org/data/definitions/119.html",
"tool": "flawfinder"
}
]
],
"remediations": [],
"scan": {
"scanner": {
"id": "gosec",
"name": "Gosec",
"url": "https://github.com/securego/gosec",
"vendor": {
"name": "GitLab"
},
"version": "2.3.0"
},
"type": "sast",
"status": "success",
"start_time": "placeholder-value",
"end_time": "placeholder-value"
}
}
ee/spec/fixtures/security_reports/master/gl-common-scanning-report.json 0 → 100644
View file @ d2b5ac90
{
"vulnerabilities": [
{
"category": "dependency_scanning",
"name": "Vulnerabilities in libxml2",
"message": "Vulnerabilities in libxml2 in nokogiri",
"description": "",
"cve": "CVE-1020",
"severity": "High",
"solution": "Upgrade to latest version.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [
{
"url": ""
}
]
},
{
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3",
"category": "dependency_scanning",
"name": "Regular Expression Denial of Service",
"message": "Regular Expression Denial of Service in debug",
"description": "",
"cve": "CVE-1030",
"severity": "Unknown",
"solution": "Upgrade to latest versions.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [
{
"url": ""
}
]
},
{
"category": "dependency_scanning",
"name": "Authentication bypass via incorrect DOM traversal and canonicalization",
"message": "Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js",
"description": "",
"cve": "yarn/yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98",
"severity": "Unknown",
"solution": "Upgrade to fixed version.\r\n",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [
{
"url": ""
},
{
"url": ""
}
]
}
],
"remediations": [
{
"fixes": [
{
"cve": "CVE-1020"
}
],
"summary": "",
"diff": ""
},
{
"fixes": [
{
"cve": "CVE",
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3"
}
],
"summary": "",
"diff": ""
},
{
"fixes": [
{
"cve": "CVE",
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3"
}
],
"summary": "",
"diff": ""
},
{
"fixes": [
{
"id": "2134",
"cve": "CVE-1"
}
],
"summary": "",
"diff": ""
}
],
"dependency_files": [],
"scan": {
"scanner": {
"id": "gemnasium",
"name": "Gemnasium",
"url": "https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven",
"vendor": {
"name": "GitLab"
},
"version": "2.18.0"
},
"type": "dependency_scanning",
"start_time": "placeholder-value",
"end_time": "placeholder-value",
"status": "success"
}
}
\ No newline at end of file
ee/spec/fixtures/security_reports/master/gl-container-scanning-report.json
View file @ d2b5ac90
......@@ -285,5 +285,18 @@
]
}
],
"remediations": []
}
"remediations": [],
"scan": {
"scanner": {
"id": "clair",
"name": "Clair",
"url": "https://github.com/coreos/clair",
"vendor": {
"name": "GitLab"
},
"version": "2.1.4"
},
"type": "container_scanning",
"status": "success"
}
}
\ No newline at end of file
ee/spec/fixtures/security_reports/master/gl-dependency-scanning-report.json
View file @ d2b5ac90
......@@ -177,5 +177,21 @@
"url": "https://github.com/ffi/ffi/releases/tag/1.9.24",
"tool": "bundler_audit"
}
]
],
"scan": {
"scanner": {
"id": "gemnasium-maven",
"name": "gemnasium-maven",
"url": "https://gitlab.com/gitlab-org/security-products/analyzers/gemnasium-maven",
"vendor": {
"name": "GitLab"
},
"version": "2.18.0"
},
"type": "dependency_scanning",
"start_time": "placeholder-value",
"end_time": "placeholder-value",
"status": "success"
}
}
ee/spec/fixtures/security_reports/master/gl-sast-report.json
View file @ d2b5ac90
......@@ -963,5 +963,21 @@
"url": "https://cwe.mitre.org/data/definitions/120.html",
"tool": "flawfinder"
}
]
],
"remediations": [],
"scan": {
"scanner": {
"id": "gosec",
"name": "Gosec",
"url": "https://github.com/securego/gosec",
"vendor": {
"name": "GitLab"
},
"version": "2.3.0"
},
"type": "sast",
"status": "success",
"start_time": "placeholder-value",
"end_time": "placeholder-value"
}
}
ee/spec/lib/gitlab/ci/parsers/security/common_spec.rb
View file @ d2b5ac90
......@@ -6,236 +6,32 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
describe '#parse!' do
let_it_be(:pipeline) { create(:ci_pipeline) }
let(:artifact) { build(:ee_ci_job_artifact, :dependency_scanning) }
let(:artifact) { build(:ee_ci_job_artifact, :common_security_report) }
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, 2.weeks.ago) }
let(:parser) { described_class.new }
before do
allow(parser).to receive(:create_location).and_return(nil)
end
it 'converts undefined severity and confidence' do
artifact.each_blob do |blob|
blob.gsub!("Unknown", "Undefined")
parser.parse!(blob, report)
end
expect(report.findings.map(&:severity)).to include("unknown")
expect(report.findings.map(&:confidence)).to include("unknown")
expect(report.findings.map(&:severity)).not_to include("undefined")
expect(report.findings.map(&:confidence)).not_to include("undefined")
end
context 'parsing scanned resources' do
describe 'when the URL is invalid' do
let(:raw_json) do
{
"vulnerabilities": [],
"remediations": [],
"dependency_files": [],
"scan": {
"scanned_resources": [
{
"method": "GET",
"type": "url",
"url": "not a URL"
}
]
}
}
end
it 'skips invalid URLs' do
parser.parse!(raw_json.to_json, report)
expect(report.scanned_resources).to be_empty
end
end
describe 'when the URLs are valid' do
let(:raw_json) do
{
"vulnerabilities": [],
"remediations": [],
"dependency_files": [],
"scan": {
"scanned_resources": [
{
"method": "GET",
"type": "url",
"url": "http://example.com/1"
},
{
"method": "POST",
"type": "url",
"url": "http://example.com/2"
},
{
"method": "GET",
"type": "url",
"url": "http://example.com/3"
}
]
}
}
end
it 'creates a scanned resource for each URL' do
parser.parse!(raw_json.to_json, report)
expect(report.scanned_resources.length).to eq(3)
end
it 'converts the JSON to Scanned Resource objects' do
parser.parse!(raw_json.to_json, report)
expect(report.scanned_resources.first).to be_a(::Gitlab::Ci::Reports::Security::ScannedResource)
end
end
end
context 'parsing remediations' do
let(:raw_json) do
{
"vulnerabilities": [
{
"category": "dependency_scanning",
"name": "Vulnerabilities in libxml2",
"message": "Vulnerabilities in libxml2 in nokogiri",
"description": "",
"cve": "CVE-1020",
"severity": "High",
"solution": "Upgrade to latest version.",
"scanner": { "id": "gemnasium", "name": "Gemnasium" },
"location": {},
"identifiers": [],
"links": [{ "url": "" }]
},
{
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3",
"category": "dependency_scanning",
"name": "Regular Expression Denial of Service",
"message": "Regular Expression Denial of Service in debug",
"description": "",
"cve": "CVE-1030",
"severity": "Unknown",
"solution": "Upgrade to latest versions.",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [{ "url": "" }]
},
{
"category": "dependency_scanning",
"name": "Authentication bypass via incorrect DOM traversal and canonicalization",
"message": "Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js",
"description": "",
"cve": "yarn/yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98",
"severity": "Unknown",
"solution": "Upgrade to fixed version.\r\n",
"scanner": {
"id": "gemnasium",
"name": "Gemnasium"
},
"location": {},
"identifiers": [],
"links": [{ "url": "" }, { "url": "" }]
}
],
"remediations": [],
"dependency_files": [],
"scan": {
"scanner": {
"id": "gemnasium",
"name": "Gemnasium",
"vendor": { "name": "GitLab" }
}
}
}
end
it 'finds remediation with same cve' do
fix_with_cve = {
"fixes": [
{
"cve": "CVE-1020"
}
],
"summary": "",
"diff": ""
}
raw_json[:remediations] << fix_with_cve
parser.parse!(raw_json.to_json, report)
vulnerability = report.findings.find { |x| x.compare_key == "CVE-1020" }
expect(vulnerability.raw_metadata).to include fix_with_cve.to_json
remediation = { 'fixes' => [{ 'cve' => 'CVE-1020' }], 'summary' => '', 'diff' => '' }
expect(Gitlab::Json.parse(vulnerability.raw_metadata).dig('remediations').first).to include remediation
end
it 'finds remediation with same id' do
fix_with_id = {
"fixes": [
{
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3",
"cve": "CVE"
}
],
"summary": "",
"diff": ""
}
raw_json[:remediations] << fix_with_id
parser.parse!(raw_json.to_json, report)
vulnerability = report.findings.find { |x| x.compare_key == "CVE-1030" }
expect(vulnerability.raw_metadata).to include fix_with_id.to_json
end
it 'finds cve and id' do
fix_with_id = {
"fixes": [
{
"id": "bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3",
"cve": "CVE"
}
],
"summary": "",
"diff": ""
}
fix_with_cve = {
"fixes": [
{
"cve": "CVE-1020"
}
],
"summary": "",
"diff": ""
}
raw_json[:remediations] << fix_with_id << fix_with_cve
parser.parse!(raw_json.to_json, report)
vulnerability_1030 = report.findings.find { |x| x.compare_key == "CVE-1030" }
expect(vulnerability_1030.raw_metadata).to include fix_with_id.to_json
vulnerability_1020 = report.findings.find { |x| x.compare_key == "CVE-1020" }
expect(vulnerability_1020.raw_metadata).to include fix_with_cve.to_json
remediation = { 'fixes' => [{ 'cve' => 'CVE', 'id' => 'bb2fbeb1b71ea360ce3f86f001d4e84823c3ffe1a1f7d41ba7466b14cfa953d3' }], 'summary' => '', 'diff' => '' }
expect(Gitlab::Json.parse(vulnerability.raw_metadata).dig('remediations').first).to include remediation
end
it 'does not find remediation with different id' do
fix_with_id = {
"fixes": [
{
"id": "bb2f",
"cve": "CVE"
}
],
"summary": "",
"diff": ""
}
fix_with_id_2 = {
"fixes": [
{
"id": "2134",
......@@ -246,55 +42,16 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Common do
"diff": ""
}
raw_json[:remediations] << fix_with_id << fix_with_id_2
parser.parse!(raw_json.to_json, report)
report.findings.map do |vulnerability|
expect(vulnerability.raw_metadata).not_to include(fix_with_id.to_json)
expect(Gitlab::Json.parse(vulnerability.raw_metadata).dig('remediations')).not_to include(fix_with_id)
end
end
end
context 'parsing scanners' do
let(:raw_json) do
{
"vulnerabilities": [
{
"category": "dependency_scanning",
"name": "Vulnerabilities in libxml2",
"message": "Vulnerabilities in libxml2 in nokogiri",
"description": "",
"cve": "CVE-1020",
"severity": "High",
"solution": "Upgrade to latest version.",
"scanner": raw_scanner,
"location": {},
"identifiers": [],
"links": [{ "url": "" }]
}
],
"remediations": [],
"dependency_files": []
}
end
subject(:scanner) { report.findings.first.scanner }
before do
parser.parse!(raw_json.to_json, report)
end
context 'when vendor is missing in scanner' do
let(:raw_scanner) { { 'id': 'gemnasium', 'name': 'Gemnasium' } }
it 'returns scanner with empty vendor field' do
expect(scanner.vendor).to be_nil
end
end
context 'when vendor is not missing in scanner' do
let(:raw_scanner) { { 'id': 'gemnasium', 'name': 'Gemnasium', 'vendor': { 'name': 'GitLab' } } }
it 'returns scanner with parsed vendor value' do
expect(scanner.vendor).to eq('GitLab')
end
......
ee/spec/lib/gitlab/ci/parsers/security/container_scanning_spec.rb
View file @ d2b5ac90
......@@ -21,7 +21,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::ContainerScanning do
it "parses all identifiers and findings for unapproved vulnerabilities" do
expect(report.findings.length).to eq(8)
expect(report.identifiers.length).to eq(8)
expect(report.scanners.length).to eq(1)
expect(report.scanners.length).to eq(2)
end
it 'generates expected location' do
......
ee/spec/lib/gitlab/ci/parsers/security/dast_spec.rb
View file @ d2b5ac90
......@@ -46,7 +46,6 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Dast do
it 'generates expected location' do
location = report.findings.last.location
expect(location).to be_a(::Gitlab::Ci::Reports::Security::Locations::Dast)
expect(location).to have_attributes(
hostname: last_occurrence_hostname,
......@@ -71,5 +70,42 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Dast do
end
end
end
describe 'parses scanned_resources' do
let(:artifact) { create(:ee_ci_job_artifact, 'dast') }
before do
artifact.each_blob do |blob|
parser.parse!(blob, report)
end
end
let(:raw_json) do
{
"vulnerabilities": [],
"remediations": [],
"dependency_files": [],
"scan": {
"scanned_resources": [
{
"method": "GET",
"type": "url",
"url": "not a URL"
}
]
}
}
end
it 'skips invalid URLs' do
parser.parse!(raw_json.to_json, report)
expect(report.scanned_resources).to be_empty
end
it 'creates a scanned resource for each URL' do
expect(report.scanned_resources.length).to eq(6)
expect(report.scanned_resources.first).to be_a(::Gitlab::Ci::Reports::Security::ScannedResource)
end
end
end
end
ee/spec/lib/gitlab/ci/parsers/security/dependency_scanning_spec.rb
View file @ d2b5ac90
......@@ -13,7 +13,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::DependencyScanning do
let(:parser) { described_class.new }
where(:report_format, :occurrence_count, :identifier_count, :scanner_count, :file_path, :package_name, :package_version, :version) do
:dependency_scanning | 4 | 7 | 2 | 'app/pom.xml' | 'io.netty/netty' | '3.9.1.Final' | '1.3'
:dependency_scanning | 4 | 7 | 3 | 'app/pom.xml' | 'io.netty/netty' | '3.9.1.Final' | '1.3'
:dependency_scanning_deprecated | 4 | 7 | 2 | 'app/pom.xml' | 'io.netty/netty' | '3.9.1.Final' | '1.3'
:dependency_scanning_remediation | 2 | 3 | 1 | 'yarn.lock' | 'debug' | '1.0.5' | '2.0'
end
......
ee/spec/lib/gitlab/ci/parsers/security/sast_spec.rb
View file @ d2b5ac90
......@@ -3,6 +3,8 @@
require 'spec_helper'
RSpec.describe Gitlab::Ci::Parsers::Security::Sast do
using RSpec::Parameterized::TableSyntax
describe '#parse!' do
let_it_be(:pipeline) { create(:ci_pipeline) }
......@@ -11,7 +13,10 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do
subject(:parser) { described_class.new }
context "when parsing valid reports" do
where(report_format: %i(sast sast_deprecated))
where(:report_format, :scanner_length) do
:sast | 4
:sast_deprecated | 3
end
with_them do
let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type, pipeline, created_at) }
......@@ -26,7 +31,7 @@ RSpec.describe Gitlab::Ci::Parsers::Security::Sast do
it "parses all identifiers and findings" do
expect(report.findings.length).to eq(33)
expect(report.identifiers.length).to eq(17)
expect(report.scanners.length).to eq(3)
expect(report.scanners.length).to eq(scanner_length)
end
it 'generates expected location' do
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment
GitLab Nexedi Edition | About GitLab | About Nexedi | 沪ICP备2021021310号-2 | 沪ICP备2021021310号-7