Merge branch '343576-data-fix-migration' into 'master'

Delete invalid vulnerability remediations

See merge request gitlab-org/gitlab!74951

db/post_migrate/20211118194239_drop_invalid_remediations.rb

+# frozen_string_literal: true
+
+class DropInvalidRemediations < Gitlab::Database::Migration[1.0]
+  BATCH_SIZE = 3_000
+  DELAY_INTERVAL = 3.minutes
+  MIGRATION_NAME = 'DropInvalidRemediations'
+  DAY_PRIOR_TO_BUG_INTRODUCTION = DateTime.new(2021, 8, 1, 0, 0, 0)
+
+  disable_ddl_transaction!
+
+  def up
+    return unless Gitlab.ee?
+
+    relation = Gitlab::BackgroundMigration::DropInvalidRemediations::FindingRemediation.where("created_at > ?", DAY_PRIOR_TO_BUG_INTRODUCTION)
+    queue_background_migration_jobs_by_range_at_intervals(relation,
+                                                          MIGRATION_NAME,
+                                                          DELAY_INTERVAL,
+                                                          batch_size: BATCH_SIZE,
+                                                          track_jobs: true)
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20211118194239

+04a4b10085bae2006ac78600b3cc410d130f9ac6944103c7bd85f71e060d4a67
\ No newline at end of file

ee/lib/ee/gitlab/background_migration/drop_invalid_remediations.rb

+# frozen_string_literal: true
+
+module EE
+  module Gitlab
+    module BackgroundMigration
+      module DropInvalidRemediations
+        class Finding < ActiveRecord::Base
+          self.table_name = 'vulnerability_occurrences'
+        end
+
+        class FindingRemediation < ActiveRecord::Base
+          include ::EachBatch
+
+          belongs_to :finding, class_name: '::Gitlab::BackgroundMigration::DropInvalidRemediations::Finding'
+          self.table_name = 'vulnerability_findings_remediations'
+        end
+
+        def perform(start_id, end_id)
+          finding_remediation_batch = FindingRemediation.where(id: start_id..end_id)
+          findings_without_remediation_ids = []
+          Finding.where(id: finding_remediation_batch.select(:vulnerability_occurrence_id)).each do |finding|
+            findings_without_remediation_ids << finding.id if ::Gitlab::Json.parse(finding[:raw_metadata])["remediations"] == [nil] rescue false
+          end
+
+          remediations_to_destroy = finding_remediation_batch.select {|finding_remediation| findings_without_remediation_ids.include? finding_remediation.vulnerability_occurrence_id}
+
+          FindingRemediation.where(id: remediations_to_destroy).each_batch(of: 100) do |batch|
+            batch.delete_all
+          end
+
+          mark_job_as_succeeded(start_id, end_id)
+        end
+
+        private
+
+        def mark_job_as_succeeded(*arguments)
+          ::Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+            self.class.name.demodulize,
+            arguments
+          )
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/drop_invalid_remediations_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::DropInvalidRemediations, schema: 20211118194239 do
+  let(:remediations) { table(:vulnerability_findings_remediations) }
+
+  let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:users) { table(:users) }
+  let(:user) { create_user! }
+  let(:project) { table(:projects).create!(id: 14219619, namespace_id: namespace.id) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let!(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:vulnerability_findings) { table(:vulnerability_occurrences) }
+  let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
+  let(:vulnerability_identifier) do
+    vulnerability_identifiers.create!(
+      id: 1244459,
+      project_id: project.id,
+      external_type: 'vulnerability-identifier',
+      external_id: 'vulnerability-identifier',
+      fingerprint: '0a203e8cd5260a1948edbedc76c7cb91ad6a2e45',
+      name: 'vulnerability identifier')
+  end
+
+  let!(:vulnerability_1) do
+    create_vulnerability!(
+      project_id: project.id,
+      author_id: user.id
+    )
+  end
+
+  let!(:vulnerability_2) do
+    create_vulnerability!(
+      project_id: project.id,
+      author_id: user.id
+    )
+  end
+
+  let!(:corrupt_finding_1) do
+    create_finding!(
+      id: 5606961,
+      uuid: "bd95c085-71aa-51d7-9bb6-08ae669c262e",
+      vulnerability_id: vulnerability_1.id,
+      report_type: 0,
+      location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed74',
+      primary_identifier_id: vulnerability_identifier.id,
+      scanner_id: scanner.id,
+      project_id: project.id,
+      raw_metadata: metadata(true)
+    )
+  end
+
+  let!(:finding_2) do
+    create_finding!(
+      id: 8765432,
+      uuid: "5b714f58-1176-5b26-8fd5-e11dfcb031b5",
+      vulnerability_id: vulnerability_2.id,
+      report_type: 0,
+      location_fingerprint: '00049d5119c2cb3bfb3d1ee1f6e031fe925aed75',
+      primary_identifier_id: vulnerability_identifier.id,
+      scanner_id: scanner.id,
+      project_id: project.id,
+      raw_metadata: metadata(false)
+    )
+  end
+
+  let!(:remediation1) { remediations.create!(vulnerability_occurrence_id: corrupt_finding_1.id) }
+  let!(:remediation2) { remediations.create!(vulnerability_occurrence_id: finding_2.id) }
+
+  context 'corresponding vuln has a remediation provided' do
+    it 'only deletes the finding_remediations without a remediation' do
+      expect { described_class.new.perform(remediation1.id, remediation2.id )}.to change {remediations.count}.from(2).to(1)
+    end
+  end
+
+  private
+
+  def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
+    vulnerabilities.create!(
+      project_id: project_id,
+      author_id: author_id,
+      title: title,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type
+    )
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+  def create_finding!(
+    id: nil,
+    vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
+                      name: "test", severity: 7, confidence: 7, report_type: 0,
+                      project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
+                      metadata_version: 'test', raw_metadata: 'test', uuid: 'test')
+    vulnerability_findings.create!(
+      vulnerability_id: vulnerability_id,
+      project_id: project_id,
+      name: name,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_fingerprint: project_fingerprint,
+      scanner_id: scanner.id,
+      primary_identifier_id: vulnerability_identifier.id,
+      location_fingerprint: location_fingerprint,
+      metadata_version: metadata_version,
+      raw_metadata: raw_metadata,
+      uuid: uuid
+    )
+  end
+  # rubocop:enable Metrics/ParameterLists
+
+  def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now)
+    users.create!(
+      name: name,
+      email: email,
+      username: name,
+      projects_limit: 0,
+      user_type: user_type,
+      confirmed_at: confirmed_at
+    )
+  end
+
+  def metadata(corrupt)
+    remediations = if corrupt
+                     [nil]
+                   else
+                     [
+                       {
+                         summary: 'summary',
+                         diff: Base64.encode64("This ain't a diff")
+                       }
+                     ]
+                   end
+
+    { remediations: remediations }.to_json
+  end
+end

ee/spec/migrations/drop_invalid_remediations_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!
+
+RSpec.describe DropInvalidRemediations, :migration do
+  let(:migration_name) { 'DropInvalidRemediations' }
+
+  let(:remediations) { table(:vulnerability_findings_remediations) }
+
+  let!(:old_remediation1) { remediations.create!( created_at: '1/1/2021') }
+  let!(:remediation1) { remediations.create! }
+  let!(:remediation2) { remediations.create! }
+  let!(:remediation3) { remediations.create! }
+  let!(:remediation4) { remediations.create! }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 2)
+    allow_next_instance_of(Gitlab::BackgroundMigration::CopyCiBuildsColumnsToSecurityScans) do |instance|
+      allow(instance).to receive(:mark_job_as_succeeded)
+    end
+  end
+
+  it 'correctly schedules background migrations' do
+    Sidekiq::Testing.fake! do
+      freeze_time do
+        migrate!
+
+        expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        expect(described_class::MIGRATION_NAME).to be_scheduled_migration_with_multiple_args(remediation1.id, remediation2.id)
+        expect(described_class::MIGRATION_NAME).to be_scheduled_migration_with_multiple_args(remediation3.id, remediation4.id)
+      end
+    end
+  end
+end

lib/gitlab/background_migration/drop_invalid_remediations.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class DropInvalidRemediations
+      def perform(start_id, stop_id)
+      end
+    end
+    # rubocop: enable Style/Documentation
+  end
+end
+
+Gitlab::BackgroundMigration::DropInvalidRemediations.prepend_mod_with('Gitlab::BackgroundMigration::DropInvalidRemediations')