Replace undefined confidence with unknown severity for occurrences

Add tests for new data
Add index for fast scan

db/post_migrate/20200506085748_update_undefined_confidence_from_occurrences.rb

+# frozen_string_literal: true
+
+class UpdateUndefinedConfidenceFromOccurrences < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  INDEX_NAME = 'index_vulnerability_occurrences_on_id_and_confidence_eq_zero'
+  DOWNTIME = false
+
+  disable_ddl_transaction!
+  BATCH_SIZE = 1_000
+  INTERVAL = 2.minutes
+
+  # 286_159 records to be updated on GitLab.com
+  def up
+    # create temporary index for undefined vulnerabilities
+    add_concurrent_index(:vulnerability_occurrences, :id, where: 'confidence = 0', name: INDEX_NAME)
+
+    return unless Gitlab.ee?
+
+    migration = Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel
+    migration_name = migration.to_s.demodulize
+    relation = migration::Occurrence.undefined_confidence
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          migration_name,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # no-op
+    # temporary index is to be dropped in a different migration in an upcoming release
+    remove_concurrent_index(:vulnerability_occurrences, :id, where: 'confidence = 0', name: INDEX_NAME)
+    # This migration can not be reversed because we can not know which records had undefined confidence
+  end
+end

db/structure.sql

@@ -10834,6 +10834,8 @@ CREATE UNIQUE INDEX index_vulnerability_occurrence_identifiers_on_unique_keys ON
 
 CREATE INDEX index_vulnerability_occurrence_pipelines_on_pipeline_id ON public.vulnerability_occurrence_pipelines USING btree (pipeline_id);
 
+CREATE INDEX index_vulnerability_occurrences_on_id_and_confidence_eq_zero ON public.vulnerability_occurrences USING btree (id) WHERE (confidence = 0);
+
 CREATE INDEX index_vulnerability_occurrences_on_primary_identifier_id ON public.vulnerability_occurrences USING btree (primary_identifier_id);
 
 CREATE INDEX index_vulnerability_occurrences_on_scanner_id ON public.vulnerability_occurrences USING btree (scanner_id);
@@ -13760,6 +13762,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200505164958
 20200505171834
 20200505172405
+20200506085748
 20200506125731
 20200507221434
 \.

ee/changelogs/unreleased/remove-undefined-from-confidence.yml

+---
+title: Replace undefined confidence with unknown severity for occurrences
+merge_request: 31200
+author:
+type: other

ee/lib/ee/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      module RemoveUndefinedOccurrenceConfidenceLevel
+        extend ::Gitlab::Utils::Override
+
+        class Occurrence < ActiveRecord::Base
+          include ::EachBatch
+
+          self.table_name = 'vulnerability_occurrences'
+
+          CONFIDENCE_LEVELS = {
+            undefined: 0,
+            unknown: 2
+          }.with_indifferent_access.freeze
+
+          enum confidence: CONFIDENCE_LEVELS
+
+          def self.undefined_confidence
+            where(confidence: Occurrence.confidences[:undefined])
+          end
+        end
+
+        override :perform
+        def perform(start_id, stop_id)
+          Occurrence.undefined_confidence
+                    .where(id: start_id..stop_id)
+                    .update_all(confidence: Occurrence.confidences[:unknown])
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/remove_undefined_occurrence_confidence_level_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel, :migration, schema: 20200506085748 do
+  let(:vulnerabilities) { table(:vulnerability_occurrences) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:projects) { table(:projects) }
+
+  it 'updates undefined Confidence level to unknown' do
+    projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
+
+    (1..3).to_a.each do |identifier_id|
+      identifiers.create!(id: identifier_id,
+                          project_id: 123,
+                          fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c' + identifier_id.to_s,
+                          external_type: 'SECURITY_ID',
+                          external_id: 'SECURITY_0',
+                          name: 'SECURITY_IDENTIFIER 0')
+    end
+
+    scanners.create!(id: 6, project_id: 123, external_id: 'clair', name: 'Security Scanner')
+
+    vul1 = vulnerabilities.create!(vuln_params(1))
+    vulnerabilities.create!(vuln_params(2))
+    vul3 = vulnerabilities.create!(vuln_params(3).merge(confidence: 2))
+
+    expect(vulnerabilities.where(confidence: 2).count). to eq(1)
+
+    described_class.new.perform(vul1.id, vul3.id)
+
+    expect(vulnerabilities.where(confidence: 2).count).to eq(3)
+  end
+
+  def vuln_params(primary_identifier_id)
+    attrs = attributes_for(:vulnerabilities_occurrence)
+
+    {
+      confidence: 0,
+      severity: 5,
+      report_type: 2,
+      project_id: 123,
+      scanner_id: 6,
+      primary_identifier_id: primary_identifier_id,
+      project_fingerprint: attrs[:project_fingerprint],
+      location_fingerprint: attrs[:location_fingerprint],
+      uuid: attrs[:uuid],
+      name: attrs[:name],
+      metadata_version: '1.3',
+      raw_metadata: attrs[:raw_metadata]
+    }
+  end
+end

ee/spec/migrations/update_undefined_confidence_from_occurrences_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20200506085748_update_undefined_confidence_from_occurrences.rb')
+
+describe UpdateUndefinedConfidenceFromOccurrences, :migration do
+  let(:vulnerabilities) { table(:vulnerability_occurrences) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:projects) { table(:projects) }
+  let(:vul1) { attributes_for(:vulnerabilities_occurrence, id: 1, report_type: 2, confidence: 5) }
+  let(:vul2) { attributes_for(:vulnerabilities_occurrence, id: 2, report_type: 2, confidence: 5) }
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 2)
+  end
+
+  it 'updates confidence levels for container scanning reports', :sidekiq_might_not_need_inline do
+    allow_any_instance_of(Gitlab).to receive(:ee?).and_return(true)
+
+    projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
+
+    identifiers.create!(id: 1,
+                        project_id: 123,
+                        fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c2',
+                        external_type: 'SECURITY_ID',
+                        external_id: 'SECURITY_0',
+                        name: 'SECURITY_IDENTIFIER 0')
+
+    identifiers.create!(id: 2,
+                        project_id: 123,
+                        fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c3',
+                        external_type: 'SECURITY_ID',
+                        external_id: 'SECURITY_0',
+                        name: 'SECURITY_IDENTIFIER 0')
+
+    scanners.create!(id: 6, project_id: 123, external_id: 'clair', name: 'Security Scanner')
+
+    vulnerabilities.create!(id: vul1[:id],
+                            confidence: 0,
+                            severity: 3,
+                            report_type: 2,
+                            project_id: 123,
+                            scanner_id: 6,
+                            primary_identifier_id: 1,
+                            project_fingerprint: vul1[:project_fingerprint],
+                            location_fingerprint: vul1[:location_fingerprint],
+                            uuid: vul1[:uuid],
+                            name: vul1[:name],
+                            metadata_version: '1.3',
+                            raw_metadata: vul1[:raw_metadata])
+
+    vulnerabilities.create!(id: vul2[:id],
+                            confidence: 2,
+                            severity: 3,
+                            report_type: 2,
+                            project_id: 123,
+                            scanner_id: 6,
+                            primary_identifier_id: 2,
+                            project_fingerprint: vul2[:project_fingerprint],
+                            location_fingerprint: vul2[:location_fingerprint],
+                            uuid: vul2[:uuid],
+                            name: vul2[:name],
+                            metadata_version: '1.3',
+                            raw_metadata: vul2[:raw_metadata])
+
+    expect(vulnerabilities.where(confidence: 0).count). to eq(1)
+
+    migrate!
+
+    expect(vulnerabilities.exists?(confidence: 0)).to be_falsy
+  end
+
+  it 'skips migration for ce' do
+    allow_any_instance_of(Gitlab).to receive(:ee?).and_return(false)
+
+    projects.create!(id: 123, namespace_id: 12, name: 'gitlab', path: 'gitlab')
+
+    identifiers.create!(id: 1,
+                        project_id: 123,
+                        fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c2',
+                        external_type: 'SECURITY_ID',
+                        external_id: 'SECURITY_0',
+                        name: 'SECURITY_IDENTIFIER 0')
+
+    scanners.create!(id: 6, project_id: 123, external_id: 'clair', name: 'Security Scanner')
+
+    vulnerabilities.create!(id: vul1[:id],
+                            confidence: 0,
+                            severity: 3,
+                            report_type: 2,
+                            project_id: 123,
+                            scanner_id: 6,
+                            primary_identifier_id: 1,
+                            project_fingerprint: vul1[:project_fingerprint],
+                            location_fingerprint: vul1[:location_fingerprint],
+                            uuid: vul1[:uuid],
+                            name: vul1[:name],
+                            metadata_version: '1.3',
+                            raw_metadata: vul1[:raw_metadata])
+
+    expect(vulnerabilities.where(confidence: 0).count). to eq(1)
+
+    migrate!
+
+    expect(vulnerabilities.exists?(confidence: 0)).to be_truthy
+  end
+end

lib/gitlab/background_migration/remove_undefined_occurrence_confidence_level.rb

+# frozen_string_literal: true
+# rubocop:disable Style/Documentation
+
+module Gitlab
+  module BackgroundMigration
+    class RemoveUndefinedOccurrenceConfidenceLevel
+      def perform(start_id, stop_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel.prepend_if_ee('EE::Gitlab::BackgroundMigration::RemoveUndefinedOccurrenceConfidenceLevel')