Commit 7b8a551a authored by Alan (Maciej) Paruszewski's avatar Alan (Maciej) Paruszewski Committed by Nick Thomas

Add CSV export for First Class Vulnerabilities

This adds to API ability to schedule CSV generation as background
job andd then download generated CSV file with Vulnerabilities.
parent c9a4c9af
......@@ -252,6 +252,10 @@
- 1
- - upload_checksum
- 1
- - vulnerability_exports_export
- 1
- - vulnerability_exports_export_deletion
- 1
- - web_hook
- 1
- - x509_certificate_revoke
......
# frozen_string_literal: true
class CreateVulnerabilityExports < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def change
create_table :vulnerability_exports do |t|
t.timestamps_with_timezone null: false
t.datetime_with_timezone :started_at
t.datetime_with_timezone :finished_at
t.string :status, limit: 255, null: false
t.string :file, limit: 255
t.bigint :project_id, null: false
t.bigint :author_id, null: false
t.integer :file_store
t.integer :format, limit: 2, null: false, default: 0
t.index %i[project_id id], unique: true
t.index %i[author_id]
end
end
end
# frozen_string_literal: true
class AddVulnerabilityExportProjectForeignKey < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
with_lock_retries do
add_foreign_key :vulnerability_exports, :projects, column: :project_id, on_delete: :cascade, index: false # rubocop:disable Migration/AddConcurrentForeignKey
end
end
def down
with_lock_retries do
remove_foreign_key :vulnerability_exports, column: :project_id
end
end
end
# frozen_string_literal: true
class AddVulnerabilityExportUserForeignKey < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
DOWNTIME = false
def up
with_lock_retries do
add_foreign_key :vulnerability_exports, :users, column: :author_id, on_delete: :cascade, index: false # rubocop:disable Migration/AddConcurrentForeignKey
end
end
def down
with_lock_retries do
remove_foreign_key :vulnerability_exports, column: :author_id
end
end
end
......@@ -4535,6 +4535,21 @@ ActiveRecord::Schema.define(version: 2020_03_19_203901) do
t.index ["updated_by_id"], name: "index_vulnerabilities_on_updated_by_id"
end
create_table "vulnerability_exports", force: :cascade do |t|
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
t.datetime_with_timezone "started_at"
t.datetime_with_timezone "finished_at"
t.string "status", limit: 255, null: false
t.string "file", limit: 255
t.bigint "project_id", null: false
t.bigint "author_id", null: false
t.integer "file_store"
t.integer "format", limit: 2, default: 0, null: false
t.index ["author_id"], name: "index_vulnerability_exports_on_author_id"
t.index ["project_id", "id"], name: "index_vulnerability_exports_on_project_id_and_id", unique: true
end
create_table "vulnerability_feedback", id: :serial, force: :cascade do |t|
t.datetime_with_timezone "created_at", null: false
t.datetime_with_timezone "updated_at", null: false
......@@ -5220,6 +5235,8 @@ ActiveRecord::Schema.define(version: 2020_03_19_203901) do
add_foreign_key "vulnerabilities", "users", column: "last_edited_by_id", name: "fk_1302949740", on_delete: :nullify
add_foreign_key "vulnerabilities", "users", column: "resolved_by_id", name: "fk_76bc5f5455", on_delete: :nullify
add_foreign_key "vulnerabilities", "users", column: "updated_by_id", name: "fk_7ac31eacb9", on_delete: :nullify
add_foreign_key "vulnerability_exports", "projects", on_delete: :cascade
add_foreign_key "vulnerability_exports", "users", column: "author_id", on_delete: :cascade
add_foreign_key "vulnerability_feedback", "ci_pipelines", column: "pipeline_id", on_delete: :nullify
add_foreign_key "vulnerability_feedback", "issues", on_delete: :nullify
add_foreign_key "vulnerability_feedback", "merge_requests", name: "fk_563ff1912e", on_delete: :nullify
......
......@@ -147,6 +147,10 @@ module EE
::License.feature_available?(:ci_cd_projects) && import_sources_enabled?
end
def first_class_vulnerabilities_available?(project)
::Feature.enabled?(:first_class_vulnerabilities, project)
end
def merge_pipelines_available?
return false unless @project.builds_enabled?
......@@ -222,10 +226,16 @@ module EE
pipeline_path: pipeline_url(pipeline),
pipeline_created: pipeline.created_at.to_s(:iso8601),
has_pipeline_data: "true"
}
}.merge(project_vulnerabilities_config(project))
end
end
def project_vulnerabilities_config(project)
return {} unless first_class_vulnerabilities_available?(project)
{ vulnerabilities_export_endpoint: api_v4_projects_vulnerability_exports_path(id: project.id) }
end
def can_create_feedback?(project, feedback_type)
feedback = Vulnerabilities::Feedback.new(project: project, feedback_type: feedback_type)
can?(current_user, :create_vulnerability_feedback, feedback)
......
......@@ -72,6 +72,7 @@ module EE
end
has_many :vulnerability_identifiers, class_name: 'Vulnerabilities::Identifier'
has_many :vulnerability_scanners, class_name: 'Vulnerabilities::Scanner'
has_many :vulnerability_exports, class_name: 'Vulnerabilities::Export'
has_many :protected_environments
has_many :software_license_policies, inverse_of: :project, class_name: 'SoftwareLicensePolicy'
......
# frozen_string_literal: true
module Vulnerabilities
class Export < ApplicationRecord
self.table_name = "vulnerability_exports"
belongs_to :project, optional: false
belongs_to :author, optional: false, class_name: 'User'
mount_uploader :file, AttachmentUploader
after_save :update_file_store, if: :saved_change_to_file?
enum format: {
csv: 0
}
validates :project, presence: true
validates :status, presence: true
validates :format, presence: true
validates :file, presence: true, if: :finished?
state_machine :status, initial: :created do
event :start do
transition created: :running
end
event :finish do
transition running: :finished
end
event :failed do
transition [:created, :running] => :failed
end
state :created
state :running
state :finished
state :failed
before_transition created: :running do |export|
export.started_at = Time.now
end
before_transition any => [:finished, :failed] do |export|
export.finished_at = Time.now
end
end
def completed?
finished? || failed?
end
def retrieve_upload(_identifier, paths)
Upload.find_by(model: self, path: paths)
end
def update_file_store
# The file.object_store is set during `uploader.store!`
# which happens after object is inserted/updated
self.update_column(:file_store, file.object_store)
end
end
end
......@@ -78,6 +78,8 @@ module Vulnerabilities
validates :metadata_version, presence: true
validates :raw_metadata, presence: true
delegate :name, to: :scanner, prefix: true, allow_nil: true
scope :report_type, -> (type) { where(report_type: report_types[type]) }
scope :ordered, -> { order(severity: :desc, confidence: :desc, id: :asc) }
......
......@@ -48,6 +48,7 @@ class Vulnerability < ApplicationRecord
scope :ordered, -> { order(severity: :desc) }
scope :with_findings, -> { includes(:findings) }
scope :with_findings_and_scanner, -> { includes(findings: :scanner) }
scope :for_projects, -> (project_ids) { where(project_id: project_ids) }
scope :with_report_types, -> (report_types) { where(report_type: report_types) }
......@@ -58,4 +59,6 @@ class Vulnerability < ApplicationRecord
def finding
findings.first
end
delegate :scanner_name, :metadata, to: :finding, prefix: true, allow_nil: true
end
......@@ -198,6 +198,7 @@ module EE
rule { can?(:read_vulnerability) }.policy do
enable :read_project_security_dashboard
enable :create_vulnerability
enable :create_vulnerability_export
enable :admin_vulnerability
enable :admin_vulnerability_issue_link
end
......
# frozen_string_literal: true
module Vulnerabilities
class ExportPolicy < BasePolicy
delegate { @subject.project }
condition(:is_author) { @user && @subject.author == @user }
condition(:exportable) { can?(:create_vulnerability_export, @subject.project) }
rule { exportable & is_author }.policy do
enable :read_vulnerability_export
end
end
end
# frozen_string_literal: true
module VulnerabilityExports
class CreateService
include Gitlab::Allowable
attr_reader :project, :author, :format
def initialize(project, author, format:)
@project = project
@author = author
@format = format
end
def execute
raise Gitlab::Access::AccessDeniedError unless can?(author, :create_vulnerability_export, project)
vulnerability_export = Vulnerabilities::Export.create(project: project, format: format, author: author)
::VulnerabilityExports::ExportWorker.perform_async(project.id, vulnerability_export.id)
vulnerability_export
end
end
end
# frozen_string_literal: true
module VulnerabilityExports
class ExportCsvService
attr_reader :vulnerabilities
def initialize(vulnerabilities_relation)
@vulnerabilities = vulnerabilities_relation
end
def csv_data(&block)
csv_builder.render(&block)
end
def csv_builder
@csv_builder ||= CsvBuilder.new(vulnerabilities.with_findings_and_scanner, header_to_value_hash)
end
private
def header_to_value_hash
{
'Scanner Type' => 'report_type',
'Scanner Name' => 'finding_scanner_name',
'Vulnerability' => 'title',
'Details' => 'description',
'Additional Info' => -> (vulnerability) { vulnerability.finding_metadata&.fetch('message', nil) },
'Severity' => 'severity',
'CVE' => -> (vulnerability) { vulnerability.finding_metadata&.fetch('cve', nil) }
}
end
end
end
......@@ -4,6 +4,7 @@
- license_management_settings_path = can?(current_user, :admin_software_license_policy, project) ? license_management_settings_path(project) : nil
- licenses_api_path = licenses_project_pipeline_path(project, pipeline) if project.feature_available?(:license_management)
- vulnerabilities_endpoint_path = expose_path(api_v4_projects_vulnerability_findings_path(id: project.id, params: { pipeline_id: pipeline.id, scope: 'dismissed' }))
- vulnerability_exports_endpoint_path = expose_path(api_v4_projects_vulnerability_exports_path(id: project.id))
- codequality_report_download_path = pipeline.downloadable_path_for_report_type(:codequality)
- if pipeline.expose_security_dashboard?
......@@ -14,6 +15,7 @@
project_id: project.id,
source_branch: pipeline.source_ref,
vulnerabilities_endpoint: vulnerabilities_endpoint_path,
vulnerability_exports_endpoint: vulnerability_exports_endpoint_path,
vulnerability_feedback_help_path: help_page_path('user/application_security/index'),
empty_state_unauthorized_svg_path: image_path('illustrations/user-not-logged-in.svg'),
empty_state_forbidden_svg_path: image_path('illustrations/lock_promotion.svg') } }
......
......@@ -7,6 +7,7 @@
#app{ data: { empty_state_svg_path: image_path('illustrations/security-dashboard_empty.svg'),
vulnerabilities_endpoint: expose_path(api_v4_projects_vulnerabilities_path(id: @project.id)),
vulnerability_exports_endpoint: expose_path(api_v4_projects_vulnerability_exports_path(id: @project.id)),
dashboard_documentation: help_page_path('user/application_security/security_dashboard/index') } }
-# Display table loading animation while Vue app loads
%table.table.gl-table
......
......@@ -591,3 +591,17 @@
:resource_boundary: :unknown
:weight: 1
:idempotent: true
- :name: vulnerability_exports_export
:feature_category: :vulnerability_management
:has_external_dependencies:
:urgency: :low
:resource_boundary: :cpu
:weight: 1
:idempotent: true
- :name: vulnerability_exports_export_deletion
:feature_category: :vulnerability_management
:has_external_dependencies:
:urgency: :low
:resource_boundary: :unknown
:weight: 1
:idempotent: true
# frozen_string_literal: true
module VulnerabilityExports
class ExportDeletionWorker
include ApplicationWorker
feature_category :vulnerability_management
idempotent!
def perform(project_id, vulnerability_export_id)
project = Project.find_by_id(project_id)
return unless project
vulnerability_export = project.vulnerability_exports.find_by_id(vulnerability_export_id)
return unless vulnerability_export
vulnerability_export.destroy!
end
end
end
# frozen_string_literal: true
module VulnerabilityExports
class ExportWorker
include ApplicationWorker
include ::Gitlab::ExclusiveLeaseHelpers
LEASE_TIMEOUT = 1.hour
feature_category :vulnerability_management
worker_resource_boundary :cpu
idempotent!
def perform(project_id, vulnerability_export_id)
project = Project.find_by_id(project_id)
return unless project
vulnerability_export = project.vulnerability_exports.find_by_id(vulnerability_export_id)
return unless vulnerability_export&.created?
return unless try_obtain_lease_for(project_id, vulnerability_export_id)
schedule_export_deletion(project_id, vulnerability_export_id)
vulnerability_export.start!
vulnerabilities = Security::VulnerabilitiesFinder.new(project).execute
generate_file_data(vulnerability_export.format, vulnerabilities) do |file|
vulnerability_export.file = file
vulnerability_export.file.filename = generate_filename(project, vulnerability_export.format)
vulnerability_export.finish!
end
rescue => error
logger.error class: self.class.name, message: error.message
vulnerability_export&.failed!
end
private
def try_obtain_lease_for(project_id, vulnerability_export_id)
Gitlab::ExclusiveLease
.new("vulnerability_exports_export:#{project_id}/#{vulnerability_export_id}", timeout: LEASE_TIMEOUT)
.try_obtain
end
def generate_file_data(format, vulnerabilities, &block)
case format
when 'csv'
VulnerabilityExports::ExportCsvService.new(vulnerabilities).csv_data(&block)
end
end
def schedule_export_deletion(project_id, vulnerability_export_id)
VulnerabilityExports::ExportDeletionWorker.perform_in(1.hour, project_id, vulnerability_export_id)
end
def generate_filename(project, format)
"#{project.full_path.parameterize}_vulnerabilities_#{Time.now.utc.strftime('%FT%H%M')}.#{format}"
end
end
end
# frozen_string_literal: true
module API
class VulnerabilityExports < Grape::API
include ::API::Helpers::VulnerabilitiesHooks
include ::Gitlab::Utils::StrongMemoize
helpers do
def vulnerability_export
strong_memoize(:vulnerability_export) do
user_project.vulnerability_exports.find(params[:export_id])
end
end
end
before do
authenticate!
end
params do
requires :id, type: String, desc: 'The ID of a project'
end
resource :projects, requirements: API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
params do
optional :export_format, type: String, desc: 'The format of export to be generated',
default: ::Vulnerabilities::Export.formats.each_key.first,
values: ::Vulnerabilities::Export.formats.keys
end
desc 'Generate an export of project vulnerability findings' do
success EE::API::Entities::VulnerabilityExport
end
before do
not_found! unless Feature.enabled?(:first_class_vulnerabilities, user_project)
end
post ':id/vulnerability_exports' do
authorize! :create_vulnerability_export, user_project
vulnerability_export = ::VulnerabilityExports::CreateService.new(
user_project, current_user, format: params[:export_format]
).execute
if vulnerability_export.persisted?
status :created
present vulnerability_export, with: EE::API::Entities::VulnerabilityExport
else
render_validation_error!(vulnerability_export)
end
end
desc 'Get single project vulnerability export' do
success EE::API::Entities::VulnerabilityExport
end
get ':id/vulnerability_exports/:export_id' do
authorize! :read_vulnerability_export, vulnerability_export
::Gitlab::PollingInterval.set_api_header(self, interval: 5_000) unless vulnerability_export.completed?
present vulnerability_export,
with: EE::API::Entities::VulnerabilityExport
end
desc 'Download single project vulnerability export'
get ':id/vulnerability_exports/:export_id/download' do
authorize! :read_vulnerability_export, vulnerability_export
if vulnerability_export.finished?
present_carrierwave_file!(vulnerability_export.file)
else
not_found!('Vulnerability Export')
end
end
end
end
end
......@@ -32,18 +32,20 @@ class CsvBuilder
# Renders the csv to a string
def render(truncate_after_bytes = nil)
tempfile = Tempfile.new('csv_export')
csv = CSV.new(tempfile)
Tempfile.open(['csv']) do |tempfile|
csv = CSV.new(tempfile)
write_csv csv, until_condition: -> do
truncate_after_bytes && tempfile.size > truncate_after_bytes
end
write_csv csv, until_condition: -> do
truncate_after_bytes && tempfile.size > truncate_after_bytes
end
tempfile.rewind
tempfile.read
ensure
tempfile.close
tempfile.unlink
if block_given?
yield tempfile
else
tempfile.rewind
tempfile.read
end
end
end
def truncated?
......
......@@ -46,6 +46,7 @@ module EE
mount ::API::Vulnerabilities
mount ::API::VulnerabilityFindings
mount ::API::VulnerabilityIssueLinks
mount ::API::VulnerabilityExports
mount ::API::MergeRequestApprovals
mount ::API::MergeRequestApprovalRules
mount ::API::ProjectAliases
......
# frozen_string_literal: true
module EE
module API
module Entities
class VulnerabilityExport < Grape::Entity
include ::API::Helpers::RelatedResourcesHelpers
expose :id
expose :created_at
expose :project_id
expose :format
expose :status
expose :started_at
expose :finished_at
expose :_links do
expose :self do |export|
expose_url api_v4_projects_vulnerability_exports_path(id: export.project_id, export_id: export.id)
end
expose :download do |export|
expose_url api_v4_projects_vulnerability_exports_download_path(id: export.project_id, export_id: export.id)
end
end
end
end
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :vulnerability_export, class: 'Vulnerabilities::Export' do
project
author
trait :csv do
format { :csv }
end
trait :with_csv_file do
file { fixture_file_upload('ee/spec/fixtures/vulnerabilities/exports/root-security-reports_vulnerabilities_2020-03-12T1235.csv') }
end
trait :created do
status { 'created' }
end
trait :running do
status { 'running' }
started_at { 1.minute.ago }
end
trait :finished do
started_at { 1.minute.ago }
finished_at { Time.now }
status { 'finished' }
end
trait :failed do
started_at { 1.minute.ago }
finished_at { Time.now }
status { 'failed' }
end
end
end
......@@ -20,6 +20,8 @@ FactoryBot.define do
raw_metadata do
{
description: "The cipher does not provide data integrity update 1",
message: "The cipher does not provide data integrity",
cve: "818bf5dacb291e15d9e6dc3c5ac32178:CIPHER",
solution: "GCM mode introduces an HMAC into the resulting encrypted data, providing integrity of the result.",
location: {
file: "maven/src/main/java/com/gitlab/security_products/tests/App.java",
......
{
"type" : "object",
"required": [
"id",
"created_at",
"project_id",
"format",
"status",
"started_at",
"finished_at",
"_links"
],
"properties" : {
"id": { "type": "integer" },
"created_at": { "type": "date" },
"project_id": { "type": "integer" },
"format": {
"type": "string",
"enum": ["csv"]
},
"status": {
"type": "string",
"enum": ["created", "running", "finished", "failed"]
},
"started_at": { "type": ["date", "null"] },
"finished_at": { "type": ["date", "null"] },
"_links": {
"type": "object",
"required": ["self", "download"],
"properties": {
"self": { "type": "string" },
"download": { "type": "string" }
},
"additionalProperties": false
}
},
"additional_properties" : false
}
Scanner Type,Scanner Name,Vulnerability,Details,Additional Info,Severity,CVE
container_scanning,Clair,CVE-2017-16997 in glibc,,CVE-2017-16997 in glibc,critical,CVE-2017-16997
container_scanning,Clair,CVE-2017-18269 in glibc,,CVE-2017-18269 in glibc,critical,CVE-2017-18269
container_scanning,Clair,CVE-2018-1000001 in glibc,,CVE-2018-1000001 in glibc,high,CVE-2018-1000001
container_scanning,Clair,CVE-2016-10228 in glibc,,CVE-2016-10228 in glibc,medium,CVE-2016-10228
container_scanning,Clair,CVE-2010-4052 in glibc,,CVE-2010-4052 in glibc,low,CVE-2010-4052
container_scanning,Clair,CVE-2018-18520 in elfutils,,CVE-2018-18520 in elfutils,low,CVE-2018-18520
container_scanning,Clair,CVE-2018-16869 in nettle,,CVE-2018-16869 in nettle,unknown,CVE-2018-16869
dependency_scanning,Gemnasium,Regular Expression Denial of Service in debug,,Regular Expression Denial of Service in debug,unknown,yarn.lock:debug:gemnasium:37283ed4-0380-40d7-ada7-2d994afcc62a
dependency_scanning,Gemnasium,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,,Authentication bypass via incorrect DOM traversal and canonicalization in saml2-js,unknown,yarn.lock:saml2-js:gemnasium:9952e574-7b5b-46fa-a270-aeb694198a98
sast,Find Security Bugs,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,818bf5dacb291e15d9e6dc3c5ac32178:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:47
sast,Find Security Bugs,Cipher with no integrity,,Cipher with no integrity,medium,e6449b89335daf53c0db4c0219bc1634:CIPHER_INTEGRITY:src/main/java/com/gitlab/security_products/tests/App.java:29
sast,Find Security Bugs,Predictable pseudorandom number generator,,Predictable pseudorandom number generator,medium,e8ff1d01f74cd372f78da8f5247d3e73:PREDICTABLE_RANDOM:src/main/java/com/gitlab/security_products/tests/App.java:41
sast,Find Security Bugs,ECB mode is insecure,,ECB mode is insecure,medium,ea0f905fc76f2739d5f10a1fd1e37a10:ECB_MODE:src/main/java/com/gitlab/security_products/tests/App.java:29
......@@ -104,6 +104,35 @@ describe ProjectsHelper do
context 'project with pipeline' do
subject { helper.project_security_dashboard_config(project, pipeline) }
it 'checks if first vulnerability class is enabled' do
expect(::Feature).to receive(:enabled?).with(:first_class_vulnerabilities, project)
subject
end
context 'when first first class vulnerabilities is enabled for project' do
before do
expect(::Feature).to receive(:enabled?).with(:first_class_vulnerabilities, project).and_return(true)
end
it 'checks if first vulnerability class is enabled' do
expect(subject[:vulnerabilities_export_endpoint]).to(
eq(
api_v4_projects_vulnerability_exports_path(id: project.id)
))
end
end
context 'when first first class vulnerabilities is disabled for project' do
before do
expect(::Feature).to receive(:enabled?).with(:first_class_vulnerabilities, project).and_return(false)
end
it 'checks if first vulnerability class is enabled' do
expect(subject).not_to have_key(:vulnerabilities_export_endpoint)
end
end
it 'returns config containing pipeline details' do
expect(subject[:security_dashboard_help_path]).to eq '/help/user/application_security/security_dashboard/index'
expect(subject[:has_pipeline_data]).to eq 'true'
......
# frozen_string_literal: true
require 'spec_helper'
describe ::EE::API::Entities::VulnerabilityExport do
let(:vulnerability_export) { create(:vulnerability_export, :finished, :csv, :with_csv_file) }
let(:entity) { described_class.new(vulnerability_export) }
subject { entity.as_json }
it 'contains vulnerability export properties' do
aggregate_failures do
expect(subject[:id]).to eq(vulnerability_export.id)
expect(subject[:created_at]).to eq(vulnerability_export.created_at)
expect(subject[:project_id]).to eq(vulnerability_export.project_id)
expect(subject[:format]).to eq(vulnerability_export.format)
expect(subject[:status]).to eq(vulnerability_export.status)
expect(subject[:started_at]).to eq(vulnerability_export.started_at)
expect(subject[:finished_at]).to eq(vulnerability_export.finished_at)
expect(subject[:_links][:self]).to end_with("api/v4/projects/#{vulnerability_export.project_id}/vulnerability_exports/#{vulnerability_export.id}")
expect(subject[:_links][:download]).to end_with("api/v4/projects/#{vulnerability_export.project_id}/vulnerability_exports/#{vulnerability_export.id}/download")
end
end
end
......@@ -28,6 +28,7 @@ describe Project do
it { is_expected.to have_many(:reviews).inverse_of(:project) }
it { is_expected.to have_many(:path_locks) }
it { is_expected.to have_many(:vulnerability_feedback) }
it { is_expected.to have_many(:vulnerability_exports) }
it { is_expected.to have_many(:audit_events).dependent(false) }
it { is_expected.to have_many(:protected_environments) }
it { is_expected.to have_many(:approvers).dependent(:destroy) }
......
# frozen_string_literal: true
require 'spec_helper'
describe Vulnerabilities::Export do
it { is_expected.to define_enum_for(:format) }
describe 'associations' do
it { is_expected.to belong_to(:project).required }
it { is_expected.to belong_to(:author).class_name('User').required }
end
describe 'validations' do
subject(:export) { build(:vulnerability_export) }
it { is_expected.to validate_presence_of(:status) }
it { is_expected.to validate_presence_of(:format) }
it { is_expected.not_to validate_presence_of(:file) }
context 'when export is finished' do
subject(:export) { build(:vulnerability_export, :finished) }
it { is_expected.to validate_presence_of(:file) }
end
end
describe '#status' do
subject(:vulnerability_export) { create(:vulnerability_export, :csv) }
around do |example|
Timecop.freeze { example.run }
end
context 'when the export is new' do
it { is_expected.to have_attributes(status: 'created') }
end
context 'when the export starts' do
before do
vulnerability_export.start!
end
it { is_expected.to have_attributes(status: 'running', started_at: Time.now) }
end
context 'when the export is running' do
context 'and it finishes' do
subject(:vulnerability_export) { create(:vulnerability_export, :csv, :with_file, :running) }
before do
vulnerability_export.finish!
end
it { is_expected.to have_attributes(status: 'finished', finished_at: Time.now) }
end
context 'and it fails' do
subject(:vulnerability_export) { create(:vulnerability_export, :csv, :running) }
before do
vulnerability_export.failed!
end
it { is_expected.to have_attributes(status: 'failed', finished_at: Time.now) }
end
end
end
describe '#completed?' do
context 'when status is created' do
subject { build(:vulnerability_export, :created) }
it { is_expected.not_to be_completed }
end
context 'when status is running' do
subject { build(:vulnerability_export, :running) }
it { is_expected.not_to be_completed }
end
context 'when status is finished' do
subject { build(:vulnerability_export, :finished) }
it { is_expected.to be_completed }
end
context 'when status is failed' do
subject { build(:vulnerability_export, :failed) }
it { is_expected.to be_completed }
end
end
end
......@@ -542,4 +542,12 @@ describe Vulnerabilities::Occurrence do
end
end
end
describe '#scanner_name' do
let(:vulnerabilities_occurrence) { create(:vulnerabilities_occurrence) }
subject(:scanner_name) { vulnerabilities_occurrence.scanner_name }
it { is_expected.to eq(vulnerabilities_occurrence.scanner.name) }
end
end
......@@ -159,4 +159,14 @@ describe Vulnerability do
it { is_expected.to eq(finding1) }
end
end
describe '#finding_scanner_name' do
let_it_be(:project) { create(:project, :with_vulnerabilities) }
let_it_be(:vulnerability) { project.vulnerabilities.first }
let_it_be(:finding) { create(:vulnerabilities_occurrence, vulnerability: vulnerability) }
subject(:finding_scanner_name) { vulnerability.finding_scanner_name }
it { is_expected.to eq(finding.scanner_name) }
end
end
......@@ -36,7 +36,7 @@ describe ProjectPolicy do
let(:additional_developer_permissions) do
%i[
admin_vulnerability_feedback read_project_security_dashboard read_feature_flag
read_vulnerability create_vulnerability admin_vulnerability
read_vulnerability create_vulnerability create_vulnerability_export admin_vulnerability
admin_vulnerability_issue_link read_merge_train
]
end
......@@ -527,6 +527,7 @@ describe ProjectPolicy do
it { is_expected.to be_disallowed(:create_vulnerability) }
it { is_expected.to be_disallowed(:admin_vulnerability) }
it { is_expected.to be_disallowed(:create_vulnerability_export) }
end
end
end
......
# frozen_string_literal: true
require 'spec_helper'
describe Vulnerabilities::ExportPolicy do
let!(:user) { create(:user) }
let!(:project) { create(:project) }
let(:vulnerability_export) { create(:vulnerability_export, :finished, :csv, :with_csv_file, project: project, author: author) }
subject { described_class.new(user, vulnerability_export) }
context 'when security dashboard is licensed' do
before do
stub_licensed_features(security_dashboard: true)
end
context 'with a user that is an author of vulnerability export' do
let(:author) { user }
context 'when user has access to vulnerabilities from the project' do
before do
project.add_developer(user)
end
it { is_expected.to be_allowed(:read_vulnerability_export) }
end
context 'when user has no access to vulnerabilities from the project' do
it { is_expected.to be_disallowed(:read_vulnerability_export) }
end
end
context 'with a user that is not an author of vulnerability export' do
let(:author) { create(:user) }
context 'when user has access to vulnerabilities from the project' do
before do
project.add_developer(user)
end
it { is_expected.to be_disallowed(:read_vulnerability_export) }
end
context 'when user has no access to vulnerabilities from the project' do
it { is_expected.to be_disallowed(:read_vulnerability_export) }
end
end
end
context 'when security dashboard is not licensed' do
before do
stub_licensed_features(security_dashboard: false)
end
context 'with a user that is an author of vulnerability export' do
let(:author) { user }
context 'when user has access to vulnerabilities from the project' do
before do
project.add_developer(user)
end
it { is_expected.to be_disallowed(:read_vulnerability_export) }
end
context 'when user has no access to vulnerabilities from the project' do
it { is_expected.to be_disallowed(:read_vulnerability_export) }
end
end
context 'with a user that is not an author of vulnerability export' do
let(:author) { create(:user) }
context 'when user has access to vulnerabilities from the project' do
before do
project.add_developer(user)
end
it { is_expected.to be_disallowed(:read_vulnerability_export) }
end
context 'when user has no access to vulnerabilities from the project' do
it { is_expected.to be_disallowed(:read_vulnerability_export) }
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe API::VulnerabilityExports do
include AccessMatchersForRequest
before do
stub_licensed_features(security_dashboard: true)
end
let_it_be(:user) { create(:user) }
let_it_be(:project) { create(:project, :with_vulnerabilities) }
let(:project_vulnerability_exports_path) { "/projects/#{project.id}/vulnerability_exports" }
let(:project_vulnerability_export_path) { "#{project_vulnerability_exports_path}/#{vulnerability_export.id}" }
describe 'POST /projects/:id/vulnerability_exports' do
let(:format) { 'csv' }
subject(:create_vulnerability_export) { post api(project_vulnerability_exports_path, user), params: { export_format: format } }
context 'with an authorized user with proper permissions' do
before do
project.add_developer(user)
end
context 'when format is csv' do
it 'returns information about new vulnerability export' do
create_vulnerability_export
expect(response).to have_gitlab_http_status(:created)
expect(response).to match_response_schema('public_api/v4/vulnerability_export', dir: 'ee')
end
it 'schedules job for export' do
expect(::VulnerabilityExports::ExportWorker).to receive(:perform_async).with(project.id, anything)
create_vulnerability_export
end
end
context 'when format is invalid' do
let(:format) { 'invalid' }
it 'returns error message' do
create_vulnerability_export
expect(response).to have_gitlab_http_status(:bad_request)
expect(json_response).to eq('error' => 'export_format does not have a valid value')
end
it 'does not schedule a job for export' do
expect(::VulnerabilityExports::ExportWorker).not_to receive(:perform_async)
create_vulnerability_export
end
end
it_behaves_like 'forbids access to vulnerability API endpoint in case of disabled features'
end
describe 'permissions' do
it { expect { create_vulnerability_export }.to be_allowed_for(:admin) }
it { expect { create_vulnerability_export }.to be_allowed_for(:owner).of(project) }
it { expect { create_vulnerability_export }.to be_allowed_for(:maintainer).of(project) }
it { expect { create_vulnerability_export }.to be_allowed_for(:developer).of(project) }
it { expect { create_vulnerability_export }.to be_allowed_for(:auditor) }
it { expect { create_vulnerability_export }.to be_denied_for(:reporter).of(project) }
it { expect { create_vulnerability_export }.to be_denied_for(:guest).of(project) }
it { expect { create_vulnerability_export }.to be_denied_for(:anonymous) }
end
end
describe 'GET /projects/:id/vulnerability_exports/:export_id' do
let_it_be(:vulnerability_export) { create(:vulnerability_export, :finished, :csv, :with_csv_file, project: project, author: user) }
subject(:get_vulnerability_export) { get api(project_vulnerability_export_path, user) }
context 'with an authorized user with proper permissions' do
before do
project.add_developer(user)
end
context 'when export is finished' do
it 'returns information about vulnerability export' do
get_vulnerability_export
expect(response).to have_gitlab_http_status(:ok)
expect(response).to match_response_schema('public_api/v4/vulnerability_export', dir: 'ee')
expect(json_response['id']).to eq vulnerability_export.id
end
it 'does not return Poll-Interval header' do
get_vulnerability_export
expect(response.headers['Poll-Interval']).to be_blank
end
end
context 'when export is running' do
let_it_be(:vulnerability_export) { create(:vulnerability_export, :running, :csv, project: project, author: user) }
it 'returns information about vulnerability export' do
get_vulnerability_export
expect(response).to have_gitlab_http_status(:ok)
expect(response).to match_response_schema('public_api/v4/vulnerability_export', dir: 'ee')
expect(json_response['id']).to eq vulnerability_export.id
end
it 'returns Poll-Interval header with value set to 5 seconds' do
get_vulnerability_export
expect(response.headers['Poll-Interval']).to eq '5000'
end
end
it_behaves_like 'forbids access to vulnerability API endpoint in case of disabled features'
end
describe 'permissions' do
context 'for export author' do
before do
project.add_developer(user)
end
it { expect { get_vulnerability_export }.to be_allowed_for(user) }
end
it { expect { get_vulnerability_export }.to be_denied_for(:admin) }
it { expect { get_vulnerability_export }.to be_denied_for(:owner).of(project) }
it { expect { get_vulnerability_export }.to be_denied_for(:maintainer).of(project) }
it { expect { get_vulnerability_export }.to be_denied_for(:developer).of(project) }
it { expect { get_vulnerability_export }.to be_denied_for(:auditor) }
it { expect { get_vulnerability_export }.to be_denied_for(:reporter).of(project) }
it { expect { get_vulnerability_export }.to be_denied_for(:guest).of(project) }
it { expect { get_vulnerability_export }.to be_denied_for(:anonymous) }
end
end
describe 'GET /projects/:id/vulnerability_exports/:export_id/download' do
let!(:vulnerability_export) { create(:vulnerability_export, :finished, :csv, :with_csv_file, project: project, author: user) }
subject(:download_vulnerability_export) { get api("#{project_vulnerability_export_path}/download", user) }
context 'with an authorized user with proper permissions' do
before do
project.add_developer(user)
end
context 'when export is running' do
let!(:vulnerability_export) { create(:vulnerability_export, :running, :csv, project: project, author: user) }
it 'renders 404' do
download_vulnerability_export
expect(response).to have_gitlab_http_status(:not_found)
expect(json_response).to eq('message' => '404 Vulnerability Export Not Found')
end
end
context 'when export is failed' do
let!(:vulnerability_export) { create(:vulnerability_export, :failed, :csv, project: project, author: user) }
it 'renders 404' do
download_vulnerability_export
expect(response).to have_gitlab_http_status(:not_found)
expect(json_response).to eq('message' => '404 Vulnerability Export Not Found')
end
end
context 'when export is finished' do
it 'renders 200 with CSV file' do
download_vulnerability_export
expect(response).to have_gitlab_http_status(:ok)
expect(response.body).to include 'Scanner Type,Scanner Name,Vulnerability,Details,Additional Info,Severity,CVE'
expect(response.headers['Poll-Interval']).to be_blank
end
end
it_behaves_like 'forbids access to vulnerability API endpoint in case of disabled features'
end
describe 'permissions' do
context 'for export author' do
before do
project.add_developer(user)
end
it { expect { download_vulnerability_export }.to be_allowed_for(user) }
end
it { expect { download_vulnerability_export }.to be_denied_for(:admin) }
it { expect { download_vulnerability_export }.to be_denied_for(:owner).of(project) }
it { expect { download_vulnerability_export }.to be_denied_for(:maintainer).of(project) }
it { expect { download_vulnerability_export }.to be_denied_for(:developer).of(project) }
it { expect { download_vulnerability_export }.to be_denied_for(:auditor) }
it { expect { download_vulnerability_export }.to be_denied_for(:reporter).of(project) }
it { expect { download_vulnerability_export }.to be_denied_for(:guest).of(project) }
it { expect { download_vulnerability_export }.to be_denied_for(:anonymous) }
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe VulnerabilityExports::CreateService do
include AccessMatchersGeneric
before do
stub_licensed_features(security_dashboard: true)
end
let_it_be(:user) { create(:user, :auditor) }
let(:group) { create(:group) }
let(:project) { create(:project, :public, group: group) }
let(:format) { 'csv' }
subject { described_class.new(project, user, format: format).execute }
describe '#execute' do
context 'when security dashboard feature is disabled' do
before do
stub_licensed_features(security_dashboard: false)
end
it 'raises an "access denied" error' do
expect { subject }.to raise_error(Gitlab::Access::AccessDeniedError)
end
end
context 'when security dashboard feature is enabled' do
it 'does not raise an "access denied" error' do
expect { subject }.not_to raise_error
end
it 'creates new Vulnerabilities::Export' do
expect { subject }.to change { Vulnerabilities::Export.count }.from(0).to(1)
end
it 'schedules ::VulnerabilityExports::ExportWorker background job' do
expect(::VulnerabilityExports::ExportWorker).to receive(:perform_async).with(project.id, anything)
subject
end
it 'returns new Vulnerabilities::Export with project and format assigned' do
expect(subject).to have_attributes(project_id: project.id, format: format)
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe VulnerabilityExports::ExportCsvService do
let_it_be(:project) { create(:project, :public) }
let_it_be(:vulnerability) { create(:vulnerability, :with_findings, project: project) }
let(:export_csv_service) { described_class.new(Vulnerability.all) }
subject(:csv) { CSV.parse(export_csv_service.csv_data, headers: true) }
context 'when block is not given' do
it 'renders csv to string' do
expect(export_csv_service.csv_data).to be_a String
end
end
context 'when block is given' do
it 'returns handle to Tempfile' do
expect(export_csv_service.csv_data { |file| file }).to be_a Tempfile
end
end
it 'includes the columns required for import' do
expect(csv.headers).to include('Scanner Type', 'Scanner Name', 'Vulnerability', 'Details', 'Additional Info',
'Severity', 'CVE')
end
it 'includes proper values for each column type' do
aggregate_failures do
expect(csv[0]['Scanner Type']).to eq vulnerability.report_type
expect(csv[0]['Scanner Name']).to eq vulnerability.finding_scanner_name
expect(csv[0]['Vulnerability']).to eq vulnerability.title
expect(csv[0]['Details']).to eq vulnerability.description
expect(csv[0]['Additional Info']).to eq vulnerability.finding_metadata['message']
expect(csv[0]['Severity']).to eq vulnerability.severity
expect(csv[0]['CVE']).to eq vulnerability.finding_metadata['cve']
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe VulnerabilityExports::ExportDeletionWorker, type: :worker do
describe '#perform' do
let_it_be(:project) { create(:project) }
let_it_be(:vulnerability_export) { create(:vulnerability_export, :finished, :csv, :with_csv_file, project: project) }
let(:worker) { described_class.new }
subject { worker.perform(project.id, vulnerability_export.id) }
context 'when vulnerability export does not exist' do
subject { worker.perform(project.id, 9999) }
it 'does not raise exception' do
expect { subject }.not_to raise_error
end
it 'does not delete any vulnerability export from database' do
expect { subject }.not_to change { Vulnerabilities::Export.count }
end
end
context 'when vulnerability export exists' do
context 'when destroy can be performed successfully' do
it 'destroys vulnerability export' do
subject
expect(Vulnerabilities::Export.find_by_id(vulnerability_export.id)).to be_nil
end
end
context 'when destroy fails' do
before do
allow_any_instance_of(Vulnerabilities::Export).to receive(:destroy!).and_raise(ActiveRecord::RecordNotFound)
end
it 'raises exception' do
expect { subject }.to raise_error(ActiveRecord::RecordNotFound)
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe VulnerabilityExports::ExportWorker, type: :worker do
describe '#perform' do
let_it_be(:project) { create(:project, :with_vulnerabilities) }
let!(:vulnerability_export) { create(:vulnerability_export, :created, :csv, project: project) }
let(:worker) { described_class.new }
before do
allow(VulnerabilityExports::ExportDeletionWorker).to receive(:perform_in)
allow(Sidekiq.logger).to receive(:error)
end
context 'when vulnerability export does not exist' do
subject { worker.perform(project.id, 9999) }
it 'does not raise any error' do
expect { subject }.not_to raise_error
end
end
context 'when vulnerability export exists' do
include_examples 'an idempotent worker' do
let(:job_args) { [project.id, vulnerability_export.id] }
context 'when export can be performed successfully' do
it 'creates new export file' do
subject
vulnerability_export.reload
expect(vulnerability_export).to be_finished
expect(vulnerability_export.file.read).to include('Scanner Type,Scanner Name,Vulnerability,Details,Additional Info,Severity,CVE')
end
it 'schedules job to delete export in 1 hour' do
expect(VulnerabilityExports::ExportDeletionWorker).to receive(:perform_in).with(1.hour, project.id, vulnerability_export.id)
subject
end
end
end
context 'when export fails' do
subject { worker.perform(project.id, vulnerability_export.id) }
before do
allow_any_instance_of(Vulnerabilities::Export).to receive(:finish!).and_raise(ActiveRecord::RecordInvalid)
end
it 'does not raise exception' do
expect { subject }.not_to raise_error
end
it 'logs error' do
expect(Sidekiq.logger).to receive(:error).with(class: described_class.name, message: anything)
subject
end
it 'sets status of the export to failed' do
expect_any_instance_of(Vulnerabilities::Export).to receive(:failed!)
subject
end
it 'schedules job to delete export in 1 hour' do
expect(VulnerabilityExports::ExportDeletionWorker).to receive(:perform_in).with(1.hour, project.id, vulnerability_export.id)
subject
end
end
end
end
end
......@@ -5,14 +5,18 @@ module Gitlab
HEADER_NAME = 'Poll-Interval'
def self.set_header(response, interval:)
if polling_enabled?
multiplier = Gitlab::CurrentSettings.polling_interval_multiplier
value = (interval * multiplier).to_i
else
value = -1
end
response.headers[HEADER_NAME] = polling_interval_value(interval).to_s
end
def self.set_api_header(context, interval:)
context.header HEADER_NAME, polling_interval_value(interval).to_s
end
def self.polling_interval_value(interval)
return -1 unless polling_enabled?
response.headers[HEADER_NAME] = value.to_s
multiplier = Gitlab::CurrentSettings.polling_interval_multiplier
(interval * multiplier).to_i
end
def self.polling_enabled?
......
......@@ -431,6 +431,7 @@ project:
- sourced_pipelines
- prometheus_metrics
- vulnerabilities
- vulnerability_exports
- vulnerability_findings
- vulnerability_feedback
- vulnerability_identifiers
......
......@@ -33,4 +33,36 @@ describe Gitlab::PollingInterval do
end
end
end
describe '.set_api_header' do
let(:context) { double(Grape::Endpoint) }
before do
allow(context).to receive(:header)
end
context 'when polling is disabled' do
before do
stub_application_setting(polling_interval_multiplier: 0)
end
it 'sets value to -1' do
expect(context).to receive(:header).with('Poll-Interval', '-1')
polling_interval.set_api_header(context, interval: 10_000)
end
end
context 'when polling is enabled' do
before do
stub_application_setting(polling_interval_multiplier: 0.33333)
end
it 'applies modifier to base interval' do
expect(context).to receive(:header).with('Poll-Interval', '3333')
polling_interval.set_api_header(context, interval: 10_000)
end
end
end
end
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment