Merge branch '212322-update-existing-vulnerabilities-occurrences-uuids' into 'master'

Recalculate UUID for all Vulnerability::Findings

See merge request gitlab-org/gitlab!47529

changelogs/unreleased/212322-update-existing-vulnerabilities-occurrences-uuids.yml

+---
+title: Recalculate UUID for all Vulnerability::Findings
+merge_request: 47529
+author:
+type: other

db/post_migrate/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences.rb

+# frozen_string_literal: true
+
+class ScheduleRecalculateUuidOnVulnerabilitiesOccurrences < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'RecalculateVulnerabilitiesOccurrencesUuid'
+  DELAY_INTERVAL = 2.minutes.to_i
+  BATCH_SIZE = 2_500
+
+  disable_ddl_transaction!
+
+  class VulnerabilitiesFinding < ActiveRecord::Base
+    include ::EachBatch
+
+    self.table_name = "vulnerability_occurrences"
+  end
+
+  def up
+    # Make sure that RemoveDuplicateVulnerabilitiesFindings has finished running
+    # so that we don't run into duplicate UUID issues
+    Gitlab::BackgroundMigration.steal('RemoveDuplicateVulnerabilitiesFindings')
+
+    say "Scheduling #{MIGRATION} jobs"
+    queue_background_migration_jobs_by_range_at_intervals(
+      VulnerabilitiesFinding,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20201112130715

+fc821ffb2e2298bfdf3555bb255e2300f3722141cadc8f89e62646fd6f22faf0
\ No newline at end of file

lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid.rb

+# frozen_string_literal: true
+
+# rubocop: disable Style/Documentation
+class Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid
+  # rubocop: disable Gitlab/NamespacedClass
+  class VulnerabilitiesIdentifier < ActiveRecord::Base
+    self.table_name = "vulnerability_identifiers"
+    has_many :primary_findings, class_name: 'VulnerabilitiesFinding', inverse_of: :primary_identifier, foreign_key: 'primary_identifier_id'
+  end
+
+  class VulnerabilitiesFinding < ActiveRecord::Base
+    self.table_name = "vulnerability_occurrences"
+    belongs_to :primary_identifier, class_name: 'VulnerabilitiesIdentifier', inverse_of: :primary_findings, foreign_key: 'primary_identifier_id'
+    REPORT_TYPES = {
+      sast: 0,
+      dependency_scanning: 1,
+      container_scanning: 2,
+      dast: 3,
+      secret_detection: 4,
+      coverage_fuzzing: 5,
+      api_fuzzing: 6
+    }.with_indifferent_access.freeze
+    enum report_type: REPORT_TYPES
+  end
+
+  class CalculateFindingUUID
+    FINDING_NAMESPACES_IDS = {
+      development: "a143e9e2-41b3-47bc-9a19-081d089229f4",
+      test: "a143e9e2-41b3-47bc-9a19-081d089229f4",
+      staging: "a6930898-a1b2-4365-ab18-12aa474d9b26",
+      production:  "58dc0f06-936c-43b3-93bb-71693f1b6570"
+    }.freeze
+
+    NAMESPACE_REGEX = /(\h{8})-(\h{4})-(\h{4})-(\h{4})-(\h{4})(\h{8})/.freeze
+    PACK_PATTERN = "NnnnnN".freeze
+
+    def self.call(value)
+      Digest::UUID.uuid_v5(namespace_id, value)
+    end
+
+    def self.namespace_id
+      namespace_uuid = FINDING_NAMESPACES_IDS.fetch(Rails.env.to_sym)
+      # Digest::UUID is broken when using an UUID in namespace_id
+      # https://github.com/rails/rails/issues/37681#issue-520718028
+      namespace_uuid.scan(NAMESPACE_REGEX).flatten.map { |s| s.to_i(16) }.pack(PACK_PATTERN)
+    end
+  end
+  # rubocop: enable Gitlab/NamespacedClass
+
+  def perform(start_id, end_id)
+    findings = VulnerabilitiesFinding
+      .joins(:primary_identifier)
+      .select(:id, :report_type, :fingerprint, :location_fingerprint, :project_id)
+      .where(id: start_id..end_id)
+
+    mappings = findings.each_with_object({}) do |finding, hash|
+      hash[finding] = { uuid: calculate_uuid_v5_for_finding(finding) }
+    end
+
+    ::Gitlab::Database::BulkUpdate.execute(%i[uuid], mappings)
+  end
+
+  private
+
+  def calculate_uuid_v5_for_finding(vulnerability_finding)
+    return unless vulnerability_finding
+
+    uuid_v5_name_components = {
+      report_type: vulnerability_finding.report_type,
+      primary_identifier_fingerprint: vulnerability_finding.fingerprint,
+      location_fingerprint: vulnerability_finding.location_fingerprint,
+      project_id: vulnerability_finding.project_id
+    }
+
+    name = uuid_v5_name_components.values.join('-')
+
+    CalculateFindingUUID.call(name)
+  end
+end

spec/lib/gitlab/background_migration/recalculate_vulnerabilities_occurrences_uuid_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::RecalculateVulnerabilitiesOccurrencesUuid, schema: 20201110110454 do
+  let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:users) { table(:users) }
+  let(:user) { create_user! }
+  let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
+  let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
+  let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
+  let(:vulnerability_identifier) do
+    vulnerability_identifiers.create!(
+      project_id: project.id,
+      external_type: 'uuid-v5',
+      external_id: 'uuid-v5',
+      fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a',
+      name: 'Identifier for UUIDv5')
+  end
+
+  let(:different_vulnerability_identifier) do
+    vulnerability_identifiers.create!(
+      project_id: project.id,
+      external_type: 'uuid-v4',
+      external_id: 'uuid-v4',
+      fingerprint: '772da93d34a1ba010bcb5efa9fb6f8e01bafcc89',
+      name: 'Identifier for UUIDv4')
+  end
+
+  let!(:vulnerability_for_uuidv4) do
+    create_vulnerability!(
+      project_id: project.id,
+      author_id: user.id
+    )
+  end
+
+  let!(:vulnerability_for_uuidv5) do
+    create_vulnerability!(
+      project_id: project.id,
+      author_id: user.id
+    )
+  end
+
+  let(:known_uuid_v5) { "77211ed6-7dff-5f6b-8c9a-da89ad0a9b60" }
+  let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" }
+  let(:desired_uuid_v5) { "3ca8ad45-6344-508b-b5e3-306a3bd6c6ba" }
+
+  subject { described_class.new.perform(finding.id, finding.id) }
+
+  context "when finding has a UUIDv4" do
+    before do
+      @uuid_v4 = create_finding!(
+        vulnerability_id: vulnerability_for_uuidv4.id,
+        project_id: project.id,
+        scanner_id: different_scanner.id,
+        primary_identifier_id: different_vulnerability_identifier.id,
+        report_type: 0, # "sast"
+        location_fingerprint: "fa18f432f1d56675f4098d318739c3cd5b14eb3e",
+        uuid: known_uuid_v4
+      )
+    end
+
+    let(:finding) { @uuid_v4 }
+
+    it "replaces it with UUIDv5" do
+      expect(vulnerabilities_findings.pluck(:uuid)).to eq([known_uuid_v4])
+
+      subject
+
+      expect(vulnerabilities_findings.pluck(:uuid)).to eq([desired_uuid_v5])
+    end
+  end
+
+  context "when finding has a UUIDv5" do
+    before do
+      @uuid_v5 = create_finding!(
+        vulnerability_id: vulnerability_for_uuidv5.id,
+        project_id: project.id,
+        scanner_id: scanner.id,
+        primary_identifier_id: vulnerability_identifier.id,
+        report_type: 0, # "sast"
+        location_fingerprint: "838574be0210968bf6b9f569df9c2576242cbf0a",
+        uuid: known_uuid_v5
+      )
+    end
+
+    let(:finding) { @uuid_v5 }
+
+    it "stays the same" do
+      expect(vulnerabilities_findings.pluck(:uuid)).to eq([known_uuid_v5])
+
+      subject
+
+      expect(vulnerabilities_findings.pluck(:uuid)).to eq([known_uuid_v5])
+    end
+  end
+
+  private
+
+  def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
+    vulnerabilities.create!(
+      project_id: project_id,
+      author_id: author_id,
+      title: title,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type
+    )
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+  def create_finding!(
+    vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
+                      name: "test", severity: 7, confidence: 7, report_type: 0,
+                      project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
+                      metadata_version: 'test', raw_metadata: 'test', uuid: 'test')
+    vulnerabilities_findings.create!(
+      vulnerability_id: vulnerability_id,
+      project_id: project_id,
+      name: name,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_fingerprint: project_fingerprint,
+      scanner_id: scanner.id,
+      primary_identifier_id: vulnerability_identifier.id,
+      location_fingerprint: location_fingerprint,
+      metadata_version: metadata_version,
+      raw_metadata: raw_metadata,
+      uuid: uuid
+    )
+  end
+  # rubocop:enable Metrics/ParameterLists
+
+  def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.zone.now, confirmed_at: Time.zone.now)
+    users.create!(
+      name: name,
+      email: email,
+      username: name,
+      projects_limit: 0,
+      user_type: user_type,
+      confirmed_at: confirmed_at
+    )
+  end
+end

spec/migrations/20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20201112130715_schedule_recalculate_uuid_on_vulnerabilities_occurrences.rb')
+
+RSpec.describe ScheduleRecalculateUuidOnVulnerabilitiesOccurrences, :migration do
+  let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:users) { table(:users) }
+  let(:user) { create_user! }
+  let(:project) { table(:projects).create!(id: 123, namespace_id: namespace.id) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:scanner) { scanners.create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
+  let(:different_scanner) { scanners.create!(project_id: project.id, external_id: 'test 2', name: 'test scanner 2') }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
+  let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
+  let(:vulnerability_identifier) do
+    vulnerability_identifiers.create!(
+      project_id: project.id,
+      external_type: 'uuid-v5',
+      external_id: 'uuid-v5',
+      fingerprint: '7e394d1b1eb461a7406d7b1e08f057a1cf11287a',
+      name: 'Identifier for UUIDv5')
+  end
+
+  let(:different_vulnerability_identifier) do
+    vulnerability_identifiers.create!(
+      project_id: project.id,
+      external_type: 'uuid-v4',
+      external_id: 'uuid-v4',
+      fingerprint: '772da93d34a1ba010bcb5efa9fb6f8e01bafcc89',
+      name: 'Identifier for UUIDv4')
+  end
+
+  let!(:vulnerability_for_uuidv4) do
+    create_vulnerability!(
+      project_id: project.id,
+      author_id: user.id
+    )
+  end
+
+  let!(:vulnerability_for_uuidv5) do
+    create_vulnerability!(
+      project_id: project.id,
+      author_id: user.id
+    )
+  end
+
+  let(:known_uuid_v4) { "b3cc2518-5446-4dea-871c-89d5e999c1ac" }
+  let!(:finding_with_uuid_v4) do
+    create_finding!(
+      vulnerability_id: vulnerability_for_uuidv4.id,
+      project_id: project.id,
+      scanner_id: different_scanner.id,
+      primary_identifier_id: different_vulnerability_identifier.id,
+      report_type: 0, # "sast"
+      location_fingerprint: "fa18f432f1d56675f4098d318739c3cd5b14eb3e",
+      uuid: known_uuid_v4
+    )
+  end
+
+  let(:known_uuid_v5) { "e7d3d99d-04bb-5771-bb44-d80a9702d0a2" }
+  let!(:finding_with_uuid_v5) do
+    create_finding!(
+      vulnerability_id: vulnerability_for_uuidv5.id,
+      project_id: project.id,
+      scanner_id: scanner.id,
+      primary_identifier_id: vulnerability_identifier.id,
+      report_type: 0, # "sast"
+      location_fingerprint: "838574be0210968bf6b9f569df9c2576242cbf0a",
+      uuid: known_uuid_v5
+    )
+  end
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+  end
+
+  around do |example|
+    freeze_time { Sidekiq::Testing.fake! { example.run } }
+  end
+
+  it 'schedules background migration' do
+    migrate!
+
+    expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+    expect(described_class::MIGRATION).to be_scheduled_migration(finding_with_uuid_v4.id, finding_with_uuid_v4.id)
+    expect(described_class::MIGRATION).to be_scheduled_migration(finding_with_uuid_v5.id, finding_with_uuid_v5.id)
+  end
+
+  private
+
+  def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
+    vulnerabilities.create!(
+      project_id: project_id,
+      author_id: author_id,
+      title: title,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type
+    )
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+  def create_finding!(
+    vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
+                      name: "test", severity: 7, confidence: 7, report_type: 0,
+                      project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
+                      metadata_version: 'test', raw_metadata: 'test', uuid: 'test')
+    vulnerabilities_findings.create!(
+      vulnerability_id: vulnerability_id,
+      project_id: project_id,
+      name: name,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_fingerprint: project_fingerprint,
+      scanner_id: scanner.id,
+      primary_identifier_id: vulnerability_identifier.id,
+      location_fingerprint: location_fingerprint,
+      metadata_version: metadata_version,
+      raw_metadata: raw_metadata,
+      uuid: uuid
+    )
+  end
+  # rubocop:enable Metrics/ParameterLists
+
+  def create_user!(name: "Example User", email: "user@example.com", user_type: nil, created_at: Time.now, confirmed_at: Time.now)
+    users.create!(
+      name: name,
+      email: email,
+      username: name,
+      projects_limit: 0,
+      user_type: user_type,
+      confirmed_at: confirmed_at
+    )
+  end
+end