Merge branch 'sk/335501-populate-vulnerability-reads' into 'master'

Populate data for vulnerability_reads from vulnerabilities

See merge request gitlab-org/gitlab!75441

db/post_migrate/20220107064845_populate_vulnerability_reads.rb

+# frozen_string_literal: true
+
+class PopulateVulnerabilityReads < Gitlab::Database::Migration[1.0]
+  BATCH_SIZE = 10_000
+  DELAY_INTERVAL = 2.minutes
+  MIGRATION_NAME = 'PopulateVulnerabilityReads'
+  SUB_BATCH_SIZE = 1_000
+
+  disable_ddl_transaction!
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      define_batchable_model('vulnerabilities'),
+      MIGRATION_NAME,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      track_jobs: true,
+      other_job_arguments: [SUB_BATCH_SIZE]
+    )
+  end
+
+  def down
+    # no-op
+  end
+end

db/schema_migrations/20220107064845

+c7c645787aadc95c77df6420f437f78aed4e7e862ea2b66e7824766b1d9f3cb5
\ No newline at end of file

lib/gitlab/background_migration/populate_vulnerability_reads.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop:disable Style/Documentation
+    class PopulateVulnerabilityReads
+      include Gitlab::Database::DynamicModelHelpers
+
+      PAUSE_SECONDS = 0.1
+
+      def perform(start_id, end_id, sub_batch_size)
+        vulnerability_model.where(id: start_id..end_id).each_batch(of: sub_batch_size) do |sub_batch|
+          first, last = sub_batch.pluck(Arel.sql('min(id), max(id)')).first
+          connection.execute(insert_query(first, last))
+
+          sleep PAUSE_SECONDS
+        end
+
+        mark_job_as_succeeded(start_id, end_id, sub_batch_size)
+      end
+
+      private
+
+      def vulnerability_model
+        define_batchable_model('vulnerabilities', connection: connection)
+      end
+
+      def connection
+        ActiveRecord::Base.connection
+      end
+
+      def insert_query(start_id, end_id)
+        <<~SQL
+          INSERT INTO vulnerability_reads (
+            vulnerability_id,
+            project_id,
+            scanner_id,
+            report_type,
+            severity,
+            state,
+            has_issues,
+            resolved_on_default_branch,
+            uuid,
+            location_image
+          )
+          SELECT
+            vulnerabilities.id,
+            vulnerabilities.project_id,
+            vulnerability_scanners.id,
+            vulnerabilities.report_type,
+            vulnerabilities.severity,
+            vulnerabilities.state,
+            CASE
+              WHEN
+                vulnerability_issue_links.vulnerability_id IS NOT NULL
+              THEN
+                true
+              ELSE
+                false
+            END
+            has_issues,
+            vulnerabilities.resolved_on_default_branch,
+            vulnerability_occurrences.uuid::uuid,
+            vulnerability_occurrences.location ->> 'image'
+          FROM
+            vulnerabilities
+          INNER JOIN vulnerability_occurrences ON vulnerability_occurrences.vulnerability_id = vulnerabilities.id
+          INNER JOIN vulnerability_scanners ON vulnerability_scanners.id = vulnerability_occurrences.scanner_id
+          LEFT JOIN vulnerability_issue_links ON vulnerability_issue_links.vulnerability_id = vulnerabilities.id
+          WHERE vulnerabilities.id BETWEEN #{start_id} AND #{end_id}
+          ON CONFLICT(vulnerability_id) DO NOTHING;
+        SQL
+      end
+
+      def mark_job_as_succeeded(*arguments)
+        Gitlab::Database::BackgroundMigrationJob.mark_all_as_succeeded(
+          self.class.name.demodulize,
+          arguments
+        )
+      end
+    end
+    # rubocop:enable Style/Documentation
+  end
+end

spec/lib/gitlab/background_migration/populate_vulnerability_reads_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::PopulateVulnerabilityReads do
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:vulnerability_reads) { table(:vulnerability_reads) }
+  let(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
+  let(:vulnerability_issue_links) { table(:vulnerability_issue_links) }
+  let(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let(:project) { table(:projects).create!(namespace_id: namespace.id) }
+  let(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
+  let(:sub_batch_size) { 1000 }
+
+  before do
+    vulnerabilities_findings.connection.execute 'ALTER TABLE vulnerability_occurrences DISABLE TRIGGER "trigger_insert_or_update_vulnerability_reads_from_occurrences"'
+    vulnerabilities.connection.execute 'ALTER TABLE vulnerabilities DISABLE TRIGGER "trigger_update_vulnerability_reads_on_vulnerability_update"'
+    vulnerability_issue_links.connection.execute 'ALTER TABLE vulnerability_issue_links DISABLE TRIGGER "trigger_update_has_issues_on_vulnerability_issue_links_update"'
+
+    10.times.each do |x|
+      vulnerability = create_vulnerability!(
+        project_id: project.id,
+        report_type: 7,
+        author_id: user.id
+      )
+      identifier = table(:vulnerability_identifiers).create!(
+        project_id: project.id,
+        external_type: 'uuid-v5',
+        external_id: 'uuid-v5',
+        fingerprint: Digest::SHA1.hexdigest("#{vulnerability.id}"),
+        name: 'Identifier for UUIDv5')
+
+      create_finding!(
+        vulnerability_id: vulnerability.id,
+        project_id: project.id,
+        scanner_id: scanner.id,
+        primary_identifier_id: identifier.id
+      )
+    end
+  end
+
+  it 'creates vulnerability_reads for the given records' do
+    described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
+
+    expect(vulnerability_reads.count).to eq(10)
+  end
+
+  it 'does not create new records when records already exists' do
+    described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
+    described_class.new.perform(vulnerabilities.first.id, vulnerabilities.last.id, sub_batch_size)
+
+    expect(vulnerability_reads.count).to eq(10)
+  end
+
+  private
+
+  def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
+    vulnerabilities.create!(
+      project_id: project_id,
+      author_id: author_id,
+      title: title,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type
+    )
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+  def create_finding!(
+    vulnerability_id: nil, project_id:, scanner_id:, primary_identifier_id:,
+    name: "test", severity: 7, confidence: 7, report_type: 0,
+    project_fingerprint: '123qweasdzxc', location: { "image" => "alpine:3.4" }, location_fingerprint: 'test',
+    metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
+    vulnerabilities_findings.create!(
+      vulnerability_id: vulnerability_id,
+      project_id: project_id,
+      name: name,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_fingerprint: project_fingerprint,
+      scanner_id: scanner_id,
+      primary_identifier_id: primary_identifier_id,
+      location: location,
+      location_fingerprint: location_fingerprint,
+      metadata_version: metadata_version,
+      raw_metadata: raw_metadata,
+      uuid: uuid
+    )
+  end
+  # rubocop:enable Metrics/ParameterLists
+end

spec/migrations/20220107064845_populate_vulnerability_reads_spec.rb

+# frozen_string_literal: true
+require 'spec_helper'
+
+require_migration!
+
+RSpec.describe PopulateVulnerabilityReads, :migration do
+  let_it_be(:namespace) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let_it_be(:user) { table(:users).create!(email: 'author@example.com', username: 'author', projects_limit: 10) }
+  let_it_be(:project) { table(:projects).create!(namespace_id: namespace.id) }
+  let_it_be(:scanner) { table(:vulnerability_scanners).create!(project_id: project.id, external_id: 'test 1', name: 'test scanner 1') }
+  let_it_be(:background_migration_jobs) { table(:background_migration_jobs) }
+  let_it_be(:vulnerabilities) { table(:vulnerabilities) }
+  let_it_be(:vulnerability_reads) { table(:vulnerability_reads) }
+  let_it_be(:vulnerabilities_findings) { table(:vulnerability_occurrences) }
+  let_it_be(:vulnerability_issue_links) { table(:vulnerability_issue_links) }
+  let_it_be(:vulnerability_ids) { [] }
+
+  before do
+    stub_const("#{described_class}::BATCH_SIZE", 1)
+    stub_const("#{described_class}::SUB_BATCH_SIZE", 1)
+
+    5.times.each do |x|
+      vulnerability = create_vulnerability!(
+        project_id: project.id,
+        report_type: 7,
+        author_id: user.id
+      )
+      identifier = table(:vulnerability_identifiers).create!(
+        project_id: project.id,
+        external_type: 'uuid-v5',
+        external_id: 'uuid-v5',
+        fingerprint: Digest::SHA1.hexdigest("#{vulnerability.id}"),
+        name: 'Identifier for UUIDv5')
+
+      create_finding!(
+        vulnerability_id: vulnerability.id,
+        project_id: project.id,
+        scanner_id: scanner.id,
+        primary_identifier_id: identifier.id
+      )
+
+      vulnerability_ids << vulnerability.id
+    end
+  end
+
+  around do |example|
+    freeze_time { Sidekiq::Testing.fake! { example.run } }
+  end
+
+  it 'schedules background migrations' do
+    migrate!
+
+    expect(background_migration_jobs.count).to eq(5)
+    expect(background_migration_jobs.first.arguments).to match_array([vulnerability_ids.first, vulnerability_ids.first, 1])
+    expect(background_migration_jobs.second.arguments).to match_array([vulnerability_ids.second, vulnerability_ids.second, 1])
+    expect(background_migration_jobs.third.arguments).to match_array([vulnerability_ids.third, vulnerability_ids.third, 1])
+    expect(background_migration_jobs.fourth.arguments).to match_array([vulnerability_ids.fourth, vulnerability_ids.fourth, 1])
+    expect(background_migration_jobs.fifth.arguments).to match_array([vulnerability_ids.fifth, vulnerability_ids.fifth, 1])
+
+    expect(BackgroundMigrationWorker.jobs.size).to eq(5)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(2.minutes, vulnerability_ids.first, vulnerability_ids.first, 1)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(4.minutes, vulnerability_ids.second, vulnerability_ids.second, 1)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(6.minutes, vulnerability_ids.third, vulnerability_ids.third, 1)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(8.minutes, vulnerability_ids.fourth, vulnerability_ids.fourth, 1)
+    expect(described_class::MIGRATION_NAME).to be_scheduled_delayed_migration(10.minutes, vulnerability_ids.fifth, vulnerability_ids.fifth, 1)
+  end
+
+  private
+
+  def create_vulnerability!(project_id:, author_id:, title: 'test', severity: 7, confidence: 7, report_type: 0)
+    vulnerabilities.create!(
+      project_id: project_id,
+      author_id: author_id,
+      title: title,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type
+    )
+  end
+
+  # rubocop:disable Metrics/ParameterLists
+  def create_finding!(
+    id: nil,
+    vulnerability_id:, project_id:, scanner_id:, primary_identifier_id:,
+    name: "test", severity: 7, confidence: 7, report_type: 0,
+    project_fingerprint: '123qweasdzxc', location_fingerprint: 'test',
+    metadata_version: 'test', raw_metadata: 'test', uuid: SecureRandom.uuid)
+    params = {
+      vulnerability_id: vulnerability_id,
+      project_id: project_id,
+      name: name,
+      severity: severity,
+      confidence: confidence,
+      report_type: report_type,
+      project_fingerprint: project_fingerprint,
+      scanner_id: scanner_id,
+      primary_identifier_id: primary_identifier_id,
+      location_fingerprint: location_fingerprint,
+      metadata_version: metadata_version,
+      raw_metadata: raw_metadata,
+      uuid: uuid
+    }
+    params[:id] = id unless id.nil?
+    vulnerabilities_findings.create!(params)
+  end
+  # rubocop:enable Metrics/ParameterLists
+end