diff --git a/ee/app/models/ee/ci/build.rb b/ee/app/models/ee/ci/build.rb index 3a96874a160dd3d0f3a6f15067fffc3aba8c03db..3fe99a2a182d3b4ebc3cd4eef54e977b32571a8f 100644 --- a/ee/app/models/ee/ci/build.rb +++ b/ee/app/models/ee/ci/build.rb @@ -12,7 +12,8 @@ module EE LICENSED_PARSER_FEATURES = { sast: :sast, dependency_scanning: :dependency_scanning, - container_scanning: :container_scanning + container_scanning: :container_scanning, + dast: :dast }.with_indifferent_access.freeze prepended do @@ -63,6 +64,9 @@ module EE next if file_type == "container_scanning" && ::Feature.disabled?(:parse_container_scanning_reports, default_enabled: false) + next if file_type == "dast" && + ::Feature.disabled?(:parse_dast_reports, default_enabled: false) + security_reports.get_report(file_type).tap do |security_report| begin next unless project.feature_available?(LICENSED_PARSER_FEATURES.fetch(file_type)) diff --git a/ee/changelogs/unreleased/7062-format-dast-output.yml b/ee/changelogs/unreleased/7062-format-dast-output.yml new file mode 100644 index 0000000000000000000000000000000000000000..d798df258d8f55d14d88c6c06aa9727be945fe91 --- /dev/null +++ b/ee/changelogs/unreleased/7062-format-dast-output.yml @@ -0,0 +1,5 @@ +--- +title: Store DAST scan results in the database +merge_request: 9192 +author: +type: added diff --git a/ee/lib/ee/gitlab/ci/parsers.rb b/ee/lib/ee/gitlab/ci/parsers.rb index 212b660fc36854c859603033cdcd74dc95fe8018..1f752d4440d74008f8429932f4d1134c7db09b82 100644 --- a/ee/lib/ee/gitlab/ci/parsers.rb +++ b/ee/lib/ee/gitlab/ci/parsers.rb @@ -12,6 +12,7 @@ module EE license_management: ::Gitlab::Ci::Parsers::LicenseManagement::LicenseManagement, dependency_scanning: ::Gitlab::Ci::Parsers::Security::DependencyScanning, container_scanning: ::Gitlab::Ci::Parsers::Security::ContainerScanning, + dast: ::Gitlab::Ci::Parsers::Security::Dast, sast: ::Gitlab::Ci::Parsers::Security::Sast }) end diff --git a/ee/lib/gitlab/ci/parsers/security/common.rb b/ee/lib/gitlab/ci/parsers/security/common.rb index cd96cc0affe39456b2185f887589a89296e04feb..bc55dc305bd96c8dad243237afd2982b84e78327 100644 --- a/ee/lib/gitlab/ci/parsers/security/common.rb +++ b/ee/lib/gitlab/ci/parsers/security/common.rb @@ -89,6 +89,10 @@ module Gitlab def generate_identifier_fingerprint(identifier) Digest::SHA1.hexdigest("#{identifier['type']}:#{identifier['value']}") end + + def generate_location_fingerprint(location) + raise NotImplementedError + end end end end diff --git a/ee/lib/gitlab/ci/parsers/security/dast.rb b/ee/lib/gitlab/ci/parsers/security/dast.rb new file mode 100644 index 0000000000000000000000000000000000000000..badf934859f0ce8cc7a43aa6ca89b5e87aa20393 --- /dev/null +++ b/ee/lib/gitlab/ci/parsers/security/dast.rb @@ -0,0 +1,156 @@ +# frozen_string_literal: true + +module Gitlab + module Ci + module Parsers + module Security + class Dast < Common + FORMAT_VERSION = '2.0'.freeze + + protected + + def parse_report(json_data) + report = super + + format_report(report) + end + + private + + def format_report(data) + { + 'vulnerabilities' => extract_vulnerabilities_from(data), + 'version' => FORMAT_VERSION + } + end + + def extract_vulnerabilities_from(data) + site = data['site'] + results = [] + + if site + host = site['@name'] + + site['alerts'].each do |vulnerability| + results += flatten_vulnerabilities(vulnerability, host) + end + end + + results + end + + def flatten_vulnerabilities(vulnerability, host) + common_vulnerability = format_vulnerability(vulnerability) + + vulnerability['instances'].map do |instance| + common_vulnerability.merge('location' => location(instance, host)) + end + end + + def format_vulnerability(vulnerability) + { + 'category' => 'dast', + 'message' => vulnerability['name'], + 'description' => sanitize(vulnerability['desc']), + 'cve' => vulnerability['pluginid'], + 'severity' => severity(vulnerability['riskcode']), + 'solution' => sanitize(vulnerability['solution']), + 'confidence' => confidence(vulnerability['confidence']), + 'scanner' => { 'id' => 'zaproxy', 'name' => 'ZAProxy' }, + 'identifiers' => [ + { + 'type' => 'ZAProxy_PluginId', + 'name' => vulnerability['name'], + 'value' => vulnerability['pluginid'], + 'url' => "https://github.com/zaproxy/zaproxy/blob/w2019-01-14/docs/scanners.md" + }, + { + 'type' => 'CWE', + 'name' => "CWE-#{vulnerability['cweid']}", + 'value' => vulnerability['cweid'], + 'url' => "https://cwe.mitre.org/data/definitions/#{vulnerability['cweid']}.html" + }, + { + 'type' => 'WASC', + 'name' => "WASC-#{vulnerability['wascid']}", + 'value' => vulnerability['wascid'], + 'url' => "http://projects.webappsec.org/w/page/13246974/Threat%20Classification%20Reference%20Grid" + } + ], + 'links' => links(vulnerability['reference']) + } + end + + def generate_location_fingerprint(location) + Digest::SHA1.hexdigest("#{location['param']} #{location['method']} #{location['path']}") + end + + # https://github.com/zaproxy/zaproxy/blob/cfb44f7e29f490d95b03830d90aadaca51a72a6a/src/scripts/templates/passive/Passive%20default%20template.js#L25 + # NOTE: ZAProxy levels: 0: info, 1: low, 2: medium, 3: high + def severity(value) + case Integer(value) + when 0 + 'ignore' + when 1 + 'low' + when 2 + 'medium' + when 3 + 'high' + else + 'unknown' + end + rescue ArgumentError + 'unknown' + end + + # NOTE: ZAProxy levels: 0: falsePositive, 1: low, 2: medium, 3: high, 4: confirmed + def confidence(value) + case Integer(value) + when 0 + 'ignore' + when 1 + 'low' + when 2 + 'medium' + when 3 + 'high' + when 4 + 'critical' + else + 'unknown' + end + rescue ArgumentError + 'unknown' + end + + def links(reference) + urls_from(reference).each_with_object([]) do |url, links| + next if url.blank? + + links << { 'url' => url } + end + end + + def urls_from(reference) + tags = reference.lines('</p>') + tags.map { |tag| sanitize(tag) } + end + + def location(instance, hostname) + { + 'param' => instance['param'], + 'method' => instance['method'], + 'hostname' => hostname, + 'path' => instance['uri'].sub(hostname, '') + } + end + + def sanitize(html_str) + ActionView::Base.full_sanitizer.sanitize(html_str) + end + end + end + end + end +end diff --git a/ee/spec/lib/gitlab/ci/parsers/security/dast_spec.rb b/ee/spec/lib/gitlab/ci/parsers/security/dast_spec.rb new file mode 100644 index 0000000000000000000000000000000000000000..8728da2a5ce452fc209560ee8d1ee5b351cd5e0b --- /dev/null +++ b/ee/spec/lib/gitlab/ci/parsers/security/dast_spec.rb @@ -0,0 +1,161 @@ +# frozen_string_literal: true + +require 'spec_helper' + +describe Gitlab::Ci::Parsers::Security::Dast do + let(:parser) { described_class.new } + + describe '#parse!' do + let(:project) { artifact.project } + let(:pipeline) { artifact.job.pipeline } + let(:artifact) { create(:ee_ci_job_artifact, :dast) } + let(:report) { Gitlab::Ci::Reports::Security::Report.new(artifact.file_type) } + + before do + artifact.each_blob do |blob| + parser.parse!(blob, report) + end + end + + it 'parses all identifiers and occurrences' do + expect(report.occurrences.length).to eq(2) + expect(report.identifiers.length).to eq(3) + expect(report.scanners.length).to eq(1) + end + + it 'generates expected location fingerprint' do + expected1 = Digest::SHA1.hexdigest('X-Content-Type-Options GET ') + expected2 = Digest::SHA1.hexdigest('X-Content-Type-Options GET /') + + expect(report.occurrences.first[:location_fingerprint]).to eq(expected1) + expect(report.occurrences.last[:location_fingerprint]).to eq(expected2) + end + + describe 'occurrence properties' do + using RSpec::Parameterized::TableSyntax + + where(:attribute, :value) do + :report_type | 'dast' + :severity | 'low' + :confidence | 'medium' + end + + with_them do + it 'saves properly occurrence' do + occurrence = report.occurrences.last + + expect(occurrence[attribute]).to eq(value) + end + end + end + end + + describe '#format_vulnerability' do + let(:parsed_report) do + JSON.parse!( + File.read( + Rails.root.join('spec/fixtures/security-reports/master/gl-dast-report.json') + ) + ) + end + + let(:file_vulnerability) { parsed_report['site']['alerts'][0] } + let(:sanitized_desc) { file_vulnerability['desc'].gsub('<p>', '').gsub('</p>', '') } + let(:sanitized_solution) { file_vulnerability['solution'].gsub('<p>', '').gsub('</p>', '') } + let(:version) { parsed_report['@version'] } + + it 'format ZAProxy vulnerability into common format' do + data = parser.send(:format_vulnerability, file_vulnerability) + + expect(data['category']).to eq('dast') + expect(data['message']).to eq('X-Content-Type-Options Header Missing') + expect(data['description']).to eq(sanitized_desc) + expect(data['cve']).to eq('10021') + expect(data['severity']).to eq('low') + expect(data['confidence']).to eq('medium') + expect(data['solution']).to eq(sanitized_solution) + expect(data['scanner']).to eq({ 'id' => 'zaproxy', 'name' => 'ZAProxy' }) + expect(data['links']).to eq([{ 'url' => 'http://msdn.microsoft.com/en-us/library/ie/gg622941%28v=vs.85%29.aspx' }, + { 'url' => 'https://www.owasp.org/index.php/List_of_useful_HTTP_headers' }]) + expect(data['identifiers'][0]).to eq({ + 'type' => 'ZAProxy_PluginId', + 'name' => 'X-Content-Type-Options Header Missing', + 'value' => '10021', + 'url' => "https://github.com/zaproxy/zaproxy/blob/w2019-01-14/docs/scanners.md" + }) + expect(data['identifiers'][1]).to eq({ + 'type' => 'CWE', + 'name' => "CWE-16", + 'value' => '16', + 'url' => "https://cwe.mitre.org/data/definitions/16.html" + }) + expect(data['identifiers'][2]).to eq({ + 'type' => 'WASC', + 'name' => "WASC-15", + 'value' => '15', + 'url' => "http://projects.webappsec.org/w/page/13246974/Threat%20Classification%20Reference%20Grid" + }) + end + end + + describe '#location' do + let(:file_vulnerability) do + JSON.parse!( + File.read( + Rails.root.join('spec/fixtures/security-reports/master/gl-dast-report.json') + ) + )['site']['alerts'][0] + end + + let(:instance) { file_vulnerability['instances'][1] } + let(:host) { 'http://bikebilly-spring-auto-devops-review-feature-br-3y2gpb.35.192.176.43.xip.io' } + + it 'format location struct' do + data = parser.send(:location, instance, host) + + expect(data['param']).to eq('X-Content-Type-Options') + expect(data['method']).to eq('GET') + expect(data['hostname']).to eq(host) + expect(data['path']).to eq('/') + end + end + + describe '#severity' do + using RSpec::Parameterized::TableSyntax + + where(:severity, :expected) do + '0' | 'ignore' + '1' | 'low' + '2' | 'medium' + '3' | 'high' + '42' | 'unknown' + '' | 'unknown' + end + + with_them do + it 'substitutes with right values' do + expect(parser.send(:severity, severity)).to eq(expected) + end + end + end + + describe '#confidence' do + using RSpec::Parameterized::TableSyntax + + where(:confidence, :expected) do + '0' | 'ignore' + '1' | 'low' + '2' | 'medium' + '3' | 'high' + '4' | 'critical' + '42' | 'unknown' + '' | 'unknown' + end + + with_them do + it 'substitutes with right values' do + expect(parser.send(:confidence, confidence)).to eq(expected) + end + end + end +end diff --git a/ee/spec/models/ci/build_spec.rb b/ee/spec/models/ci/build_spec.rb index 82f8b22fa31346e18960b074433c13f5e86e3dc3..56a3c56bc53468fc0e2179924b3f68c6cb76fbfb 100644 --- a/ee/spec/models/ci/build_spec.rb +++ b/ee/spec/models/ci/build_spec.rb @@ -157,7 +157,7 @@ describe Ci::Build do subject { job.collect_security_reports!(security_reports) } before do - stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true) + stub_licensed_features(sast: true, dependency_scanning: true, container_scanning: true, dast: true) end context 'when build has a security report' do @@ -178,6 +178,7 @@ describe Ci::Build do create(:ee_ci_job_artifact, :sast, job: job, project: job.project) create(:ee_ci_job_artifact, :dependency_scanning, job: job, project: job.project) create(:ee_ci_job_artifact, :container_scanning, job: job, project: job.project) + create(:ee_ci_job_artifact, :dast, job: job, project: job.project) end it 'parses blobs and add the results to the reports' do @@ -186,6 +187,7 @@ describe Ci::Build do expect(security_reports.get_report('sast').occurrences.size).to eq(33) expect(security_reports.get_report('dependency_scanning').occurrences.size).to eq(4) expect(security_reports.get_report('container_scanning').occurrences.size).to eq(8) + expect(security_reports.get_report('dast').occurrences.size).to eq(2) end end @@ -217,6 +219,20 @@ describe Ci::Build do end end + context 'when Feature flag is disabled for DAST reports parsing' do + before do + stub_feature_flags(parse_dast_reports: false) + create(:ee_ci_job_artifact, :sast, job: job, project: job.project) + create(:ee_ci_job_artifact, :dast, job: job, project: job.project) + end + + it 'does NOT parse dast report' do + subject + + expect(security_reports.reports.keys).to contain_exactly('sast') + end + end + context 'when there is a corrupted sast report' do before do create(:ee_ci_job_artifact, :sast_with_corrupted_data, job: job, project: job.project)