Commit 5e85ec65 authored by Tiger Watson's avatar Tiger Watson

Merge branch 'remove-undefined-from-confidence' into 'master'

Replace undefined confidence with unknown severity for occurrences

See merge request gitlab-org/gitlab!31200
parents ff5ef5c6 7ea141be
# frozen_string_literal: true
class UpdateUndefinedConfidenceFromOccurrences < ActiveRecord::Migration[6.0]
include Gitlab::Database::MigrationHelpers
INDEX_NAME = 'index_vulnerability_occurrences_on_id_and_confidence_eq_zero'
DOWNTIME = false
disable_ddl_transaction!
BATCH_SIZE = 1_000
INTERVAL = 2.minutes
# 286_159 records to be updated on GitLab.com
def up
# create temporary index for undefined vulnerabilities
add_concurrent_index(:vulnerability_occurrences, :id, where: 'confidence = 0', name: INDEX_NAME)
return unless Gitlab.ee?
migration = Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel
migration_name = migration.to_s.demodulize
relation = migration::Occurrence.undefined_confidence
queue_background_migration_jobs_by_range_at_intervals(relation,
migration_name,
INTERVAL,
batch_size: BATCH_SIZE)
end
def down
# no-op
# temporary index is to be dropped in a different migration in an upcoming release
remove_concurrent_index(:vulnerability_occurrences, :id, where: 'confidence = 0', name: INDEX_NAME)
# This migration can not be reversed because we can not know which records had undefined confidence
end
end
......@@ -10834,6 +10834,8 @@ CREATE UNIQUE INDEX index_vulnerability_occurrence_identifiers_on_unique_keys ON
CREATE INDEX index_vulnerability_occurrence_pipelines_on_pipeline_id ON public.vulnerability_occurrence_pipelines USING btree (pipeline_id);
CREATE INDEX index_vulnerability_occurrences_on_id_and_confidence_eq_zero ON public.vulnerability_occurrences USING btree (id) WHERE (confidence = 0);
CREATE INDEX index_vulnerability_occurrences_on_primary_identifier_id ON public.vulnerability_occurrences USING btree (primary_identifier_id);
CREATE INDEX index_vulnerability_occurrences_on_scanner_id ON public.vulnerability_occurrences USING btree (scanner_id);
......@@ -13760,6 +13762,7 @@ COPY "schema_migrations" (version) FROM STDIN;
20200505164958
20200505171834
20200505172405
20200506085748
20200506125731
20200507221434
\.
......
---
title: Replace undefined confidence with unknown severity for occurrences
merge_request: 31200
author:
type: other
# frozen_string_literal: true
module EE
module Gitlab
module BackgroundMigration
module RemoveUndefinedOccurrenceConfidenceLevel
extend ::Gitlab::Utils::Override
class Occurrence < ActiveRecord::Base
include ::EachBatch
self.table_name = 'vulnerability_occurrences'
CONFIDENCE_LEVELS = {
undefined: 0,
unknown: 2
}.with_indifferent_access.freeze
enum confidence: CONFIDENCE_LEVELS
def self.undefined_confidence
where(confidence: Occurrence.confidences[:undefined])
end
end
override :perform
def perform(start_id, stop_id)
Occurrence.undefined_confidence
.where(id: start_id..stop_id)
.update_all(confidence: Occurrence.confidences[:unknown])
end
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
describe Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel, :migration, schema: 20200506085748 do
let(:vulnerabilities) { table(:vulnerability_occurrences) }
let(:identifiers) { table(:vulnerability_identifiers) }
let(:scanners) { table(:vulnerability_scanners) }
let(:projects) { table(:projects) }
it 'updates undefined Confidence level to unknown' do
projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
(1..3).to_a.each do |identifier_id|
identifiers.create!(id: identifier_id,
project_id: 123,
fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c' + identifier_id.to_s,
external_type: 'SECURITY_ID',
external_id: 'SECURITY_0',
name: 'SECURITY_IDENTIFIER 0')
end
scanners.create!(id: 6, project_id: 123, external_id: 'clair', name: 'Security Scanner')
vul1 = vulnerabilities.create!(vuln_params(1))
vulnerabilities.create!(vuln_params(2))
vul3 = vulnerabilities.create!(vuln_params(3).merge(confidence: 2))
expect(vulnerabilities.where(confidence: 2).count). to eq(1)
described_class.new.perform(vul1.id, vul3.id)
expect(vulnerabilities.where(confidence: 2).count).to eq(3)
end
def vuln_params(primary_identifier_id)
attrs = attributes_for(:vulnerabilities_occurrence)
{
confidence: 0,
severity: 5,
report_type: 2,
project_id: 123,
scanner_id: 6,
primary_identifier_id: primary_identifier_id,
project_fingerprint: attrs[:project_fingerprint],
location_fingerprint: attrs[:location_fingerprint],
uuid: attrs[:uuid],
name: attrs[:name],
metadata_version: '1.3',
raw_metadata: attrs[:raw_metadata]
}
end
end
# frozen_string_literal: true
require 'spec_helper'
require Rails.root.join('db', 'post_migrate', '20200506085748_update_undefined_confidence_from_occurrences.rb')
describe UpdateUndefinedConfidenceFromOccurrences, :migration do
let(:vulnerabilities) { table(:vulnerability_occurrences) }
let(:identifiers) { table(:vulnerability_identifiers) }
let(:scanners) { table(:vulnerability_scanners) }
let(:projects) { table(:projects) }
let(:vul1) { attributes_for(:vulnerabilities_occurrence, id: 1, report_type: 2, confidence: 5) }
let(:vul2) { attributes_for(:vulnerabilities_occurrence, id: 2, report_type: 2, confidence: 5) }
before do
stub_const("#{described_class}::BATCH_SIZE", 2)
end
it 'updates confidence levels for container scanning reports', :sidekiq_might_not_need_inline do
allow_any_instance_of(Gitlab).to receive(:ee?).and_return(true)
projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
identifiers.create!(id: 1,
project_id: 123,
fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c2',
external_type: 'SECURITY_ID',
external_id: 'SECURITY_0',
name: 'SECURITY_IDENTIFIER 0')
identifiers.create!(id: 2,
project_id: 123,
fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c3',
external_type: 'SECURITY_ID',
external_id: 'SECURITY_0',
name: 'SECURITY_IDENTIFIER 0')
scanners.create!(id: 6, project_id: 123, external_id: 'clair', name: 'Security Scanner')
vulnerabilities.create!(id: vul1[:id],
confidence: 0,
severity: 3,
report_type: 2,
project_id: 123,
scanner_id: 6,
primary_identifier_id: 1,
project_fingerprint: vul1[:project_fingerprint],
location_fingerprint: vul1[:location_fingerprint],
uuid: vul1[:uuid],
name: vul1[:name],
metadata_version: '1.3',
raw_metadata: vul1[:raw_metadata])
vulnerabilities.create!(id: vul2[:id],
confidence: 2,
severity: 3,
report_type: 2,
project_id: 123,
scanner_id: 6,
primary_identifier_id: 2,
project_fingerprint: vul2[:project_fingerprint],
location_fingerprint: vul2[:location_fingerprint],
uuid: vul2[:uuid],
name: vul2[:name],
metadata_version: '1.3',
raw_metadata: vul2[:raw_metadata])
expect(vulnerabilities.where(confidence: 0).count). to eq(1)
migrate!
expect(vulnerabilities.exists?(confidence: 0)).to be_falsy
end
it 'skips migration for ce' do
allow_any_instance_of(Gitlab).to receive(:ee?).and_return(false)
projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
identifiers.create!(id: 1,
project_id: 123,
fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c2',
external_type: 'SECURITY_ID',
external_id: 'SECURITY_0',
name: 'SECURITY_IDENTIFIER 0')
scanners.create!(id: 6, project_id: 123, external_id: 'clair', name: 'Security Scanner')
vulnerabilities.create!(id: vul1[:id],
confidence: 0,
severity: 3,
report_type: 2,
project_id: 123,
scanner_id: 6,
primary_identifier_id: 1,
project_fingerprint: vul1[:project_fingerprint],
location_fingerprint: vul1[:location_fingerprint],
uuid: vul1[:uuid],
name: vul1[:name],
metadata_version: '1.3',
raw_metadata: vul1[:raw_metadata])
expect(vulnerabilities.where(confidence: 0).count). to eq(1)
migrate!
expect(vulnerabilities.exists?(confidence: 0)).to be_truthy
end
end
# frozen_string_literal: true
# rubocop:disable Style/Documentation
module Gitlab
module BackgroundMigration
class RemoveUndefinedOccurrenceConfidenceLevel
def perform(start_id, stop_id)
end
end
end
end
Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel')
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment