Merge branch '217600-migrate-dismissed_data-for-vulnerabilities' into 'master'

Migrate dismissed data from dismissal feedback to Vulnerabilities

See merge request gitlab-org/gitlab!32444

db/post_migrate/20200519201128_migrate_vulnerability_dismissal_feedback.rb

+# frozen_string_literal: true
+
+class MigrateVulnerabilityDismissalFeedback < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  disable_ddl_transaction!
+
+  MIGRATION = 'UpdateVulnerabilitiesFromDismissalFeedback'
+  BATCH_SIZE = 500
+  DELAY_INTERVAL = 2.minutes.to_i
+
+  class Vulnerability < ActiveRecord::Base
+    self.table_name = 'vulnerabilities'
+    self.inheritance_column = :_type_disabled
+
+    include ::EachBatch
+  end
+
+  def up
+    return unless Gitlab.ee?
+
+    Vulnerability.select('project_id').group(:project_id).each_batch(of: BATCH_SIZE, column: "project_id") do |project_batch, index|
+      batch_delay = (index - 1) * BATCH_SIZE * DELAY_INTERVAL
+
+      project_batch.each_with_index do |project, project_batch_index|
+        project_delay = project_batch_index * DELAY_INTERVAL
+        migrate_in(batch_delay + project_delay, MIGRATION, project[:project_id])
+      end
+    end
+  end
+
+  def down
+    # nothing to do
+  end
+end

db/structure.sql

@@ -14032,6 +14032,7 @@ COPY "schema_migrations" (version) FROM STDIN;
 20200519141534
 20200519171058
 20200519194042
+20200519201128
 20200520103514
 20200521022725
 20200521225327

ee/changelogs/unreleased/217600-migrate-dismissed_data-for-vulnerabilities.yml

+---
+title: Migrate dismissal data from Vulnerability Feedback to Vulnerabilities
+merge_request: 32444
+author:
+type: fixed

ee/lib/ee/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      # This migration updates the dismissed_by_id and dismissed_at properties
+      # of dimissed vulnerabilities records
+      module UpdateVulnerabilitiesFromDismissalFeedback
+        extend ::Gitlab::Utils::Override
+
+        VULNERABILITY_DISMISSED_STATE = 2
+        VULNERABILITY_FEEDBACK_DISMISSAL_TYPE = 0
+
+        class Project < ActiveRecord::Base
+          self.table_name = 'projects'
+          self.inheritance_column = :_type_disabled
+        end
+
+        override :perform
+        def perform(project_id)
+          project = Project.find_by(id: project_id)
+
+          return unless project
+          return if project.pending_delete?
+
+          update_vulnerability_from_dismissal_feedback(project.id)
+        end
+
+        private
+
+        def update_vulnerability_from_dismissal_feedback(project_id)
+          update_vulnerability_from_dismissal_feedback_sql = <<-SQL
+          UPDATE vulnerabilities AS v
+          SET dismissed_by_id = vf.author_id, dismissed_at = vf.created_at
+          FROM vulnerability_occurrences AS vo, vulnerability_feedback AS vf
+          WHERE vo.vulnerability_id = v.id
+            AND v.state = #{VULNERABILITY_DISMISSED_STATE}
+            AND vo.project_id = vf.project_id
+            AND ENCODE(vo.project_fingerprint, 'HEX') = vf.project_fingerprint
+            AND vo.project_id = #{project_id}
+            AND vo.report_type = vf.category
+            AND vf.feedback_type = #{VULNERABILITY_FEEDBACK_DISMISSAL_TYPE};
+          SQL
+          connection.execute(update_vulnerability_from_dismissal_feedback_sql)
+        rescue => e
+          logger.warn(
+            message: 'update_vulnerability_from_dismissal_feedback errored out',
+            project_id: project_id,
+            error: e.message
+          )
+        end
+
+        def connection
+          @connection ||= ActiveRecord::Base.connection
+        end
+
+        def logger
+          @logger ||= ::Gitlab::BackgroundMigration::Logger.build
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+describe Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback, :migration, schema: 20200519201128 do
+  let(:users) { table(:users) }
+  let(:projects) { table(:projects) }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:vulnerability_occurrences) { table(:vulnerability_occurrences) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:identifiers) { table(:vulnerability_identifiers) }
+  let(:feedback) { table(:vulnerability_feedback) }
+  let(:namespaces) { table(:namespaces)}
+
+  let(:severity) { Vulnerabilities::Occurrence::SEVERITY_LEVELS[:unknown] }
+  let(:confidence) { Vulnerabilities::Occurrence::CONFIDENCE_LEVELS[:medium] }
+  let(:report_type) { Vulnerabilities::Occurrence::REPORT_TYPES[:sast] }
+
+  let!(:user) { users.create!(email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let!(:project) { projects.create!(namespace_id: namespace.id, name: 'gitlab', path: 'gitlab') }
+
+  let(:namespace) do
+    namespaces.create!(name: 'namespace', path: '/path', description: 'description')
+  end
+
+  let(:scanner) do
+    scanners.create!(project_id: project.id, external_id: 'clair', name: 'Security Scanner')
+  end
+
+  let(:identifier) do
+    identifiers.create!(project_id: project.id,
+      fingerprint: 'd432c2ad2953e8bd587a3a43b3ce309b5b0154c7',
+      external_type: 'SECURITY_ID',
+      external_id: 'SECURITY_0',
+      name: 'SECURITY_IDENTIFIER 0')
+  end
+
+  context 'vulnerability has been dismissed' do
+    let!(:vulnerability) { vulnerabilities.create!(vuln_params) }
+    let!(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: :success, user_id: user.id) }
+
+    let!(:vulnerability_occurrence) do
+      vulnerability_occurrences.create!(
+        report_type: vulnerability.report_type, name: 'finding_1',
+        primary_identifier_id: identifier.id, uuid: 'abc', project_fingerprint: 'abc123',
+        location_fingerprint: 'abc456', project_id: project.id, scanner_id: scanner.id, severity: severity,
+        confidence: confidence, metadata_version: 'sast:1.0', raw_metadata: '{}', vulnerability_id: vulnerability.id)
+    end
+
+    let!(:dismiss_feedback) do
+      feedback.create!(category: vulnerability_occurrence.report_type, feedback_type: 0,
+              project_id: project.id, project_fingerprint: vulnerability_occurrence.project_fingerprint.unpack1('H*'),
+              author_id: user.id)
+    end
+
+    it 'vulnerability should now have a dismissed_by_id' do
+      expect(vulnerability.dismissed_by_id).to eq(nil)
+
+      expect { described_class.new.perform(project.id) }
+        .to change { vulnerability.reload.dismissed_by_id }
+        .from(nil)
+        .to(dismiss_feedback.author_id)
+    end
+
+    it 'vulnerability should now have a dismissed_at' do
+      expect(vulnerability.dismissed_at).to eq(nil)
+
+      expect { described_class.new.perform(project.id) }
+        .to change { vulnerability.reload.dismissed_at }
+        .from(nil)
+        .to(dismiss_feedback.created_at)
+    end
+
+    context 'project is set to be deleted' do
+      let!(:project) { projects.create!(namespace_id: namespace.id, name: 'gitlab', path: 'gitlab', pending_delete: true) }
+
+      it 'vulnerability dismissed_by_id should remain nil' do
+        expect(vulnerability.dismissed_by_id).to eq(nil)
+
+        expect { described_class.new.perform(project.id) }.not_to change { vulnerability.reload.dismissed_by_id }.from(nil)
+      end
+
+      it 'vulnerability dismissed_at should remain nil' do
+        expect(vulnerability.dismissed_at).to eq(nil)
+
+        expect { described_class.new.perform(project.id) }.not_to change { vulnerability.reload.dismissed_at }.from(nil)
+      end
+    end
+  end
+
+  context 'has not been dismissed' do
+    let!(:vulnerability) { vulnerabilities.create!(vuln_params.merge({ state: 1 })) }
+
+    it 'vulnerability should not have a dismissed_by_id' do
+      expect(vulnerability.dismissed_by_id).to be_nil
+
+      expect { described_class.new.perform(project.id) }.not_to change { vulnerability.reload.dismissed_by_id }.from(nil)
+    end
+
+    it 'vulnerability should not have a dismissed_at' do
+      expect(vulnerability.dismissed_at).to be_nil
+
+      expect { described_class.new.perform(project.id) }.not_to change { vulnerability.reload.dismissed_at }.from(nil)
+    end
+  end
+
+  def vuln_params
+    {
+      title: 'title',
+      state: described_class::VULNERABILITY_DISMISSED_STATE,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_id: project.id,
+      author_id: user.id
+    }
+  end
+end

ee/spec/migrations/migrate_vulnerability_dismissal_feedback_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200519201128_migrate_vulnerability_dismissal_feedback.rb')
+
+describe MigrateVulnerabilityDismissalFeedback, :migration, :sidekiq do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+
+  let!(:user) { users.create!(id: 13, email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let(:namespace) { namespaces.create!(name: 'gitlab', path: 'gitlab-org') }
+  let(:project_1) { projects.create!(name: 'gitlab', path: 'gitlab-ce', namespace_id: namespace.id) }
+  let(:project_2) { projects.create!(name: 'gitlab2', path: 'gitlab-ce', namespace_id: namespace.id) }
+
+  let(:vulnerabilities) { table(:vulnerabilities) }
+
+  let(:dismissed_state) { Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback::VULNERABILITY_DISMISSED_STATE }
+  let(:severity) { Vulnerabilities::Occurrence::SEVERITY_LEVELS[:unknown] }
+  let(:confidence) { Vulnerabilities::Occurrence::CONFIDENCE_LEVELS[:medium] }
+  let(:report_type) { Vulnerabilities::Occurrence::REPORT_TYPES[:sast] }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+
+    vulnerabilities.create!(vuln_params.merge!({ project_id: project_1.id }) )
+    vulnerabilities.create!(vuln_params.merge!({ project_id: project_2.id }) )
+    vulnerabilities.create!(vuln_params.merge!({ project_id: project_2.id }) )
+  end
+
+  context 'EE' do
+    before do
+      allow(Gitlab).to receive(:ee?).and_return(true)
+    end
+
+    it 'creates background job for each project' do
+      migrate!
+
+      expect(BackgroundMigrationWorker.jobs.size).to eq 2
+    end
+
+    it 'calls the UpdateVulnerabilitiesFromDismissalFeedback migration' do
+      expect(BackgroundMigrationWorker).to receive(:perform_in).with(0, 'UpdateVulnerabilitiesFromDismissalFeedback', project_1.id )
+      expect(BackgroundMigrationWorker).to receive(:perform_in).with(120, 'UpdateVulnerabilitiesFromDismissalFeedback', project_2.id )
+
+      migrate!
+    end
+  end
+
+  context 'FOSS' do
+    before do
+      allow(Gitlab).to receive(:ee?).and_return(false)
+    end
+
+    it 'skips migration for FOSS' do
+      Sidekiq::Testing.fake! do
+        migrate!
+
+        expect(BackgroundMigrationWorker.jobs.size).to eq 0
+      end
+    end
+  end
+
+  def vuln_params
+    {
+      title: 'title',
+      state: dismissed_state,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      author_id: user.id
+    }
+  end
+end

lib/gitlab/background_migration/update_vulnerabilities_from_dismissal_feedback.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class UpdateVulnerabilitiesFromDismissalFeedback
+      def perform(project_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback.prepend_if_ee('EE::Gitlab::BackgroundMigration::UpdateVulnerabilitiesFromDismissalFeedback')