Merge branch...

Merge branch '227114_populate_resolved_on_default_branch_column_for_existing_vulnerabilities' into 'master'

Populate `resolved_on_default_branch` column for existing vulnerabilities

See merge request gitlab-org/gitlab!38795

db/post_migrate/20200806100713_schedule_populate_resolved_on_default_branch_column.rb

+# frozen_string_literal: true
+
+class SchedulePopulateResolvedOnDefaultBranchColumn < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  BATCH_SIZE = 100
+  DELAY_INTERVAL = 5.minutes.to_i
+  MIGRATION_CLASS = 'PopulateResolvedOnDefaultBranchColumn'
+  BASE_MODEL = EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::Vulnerability
+
+  disable_ddl_transaction!
+
+  def up
+    return unless run_migration?
+
+    BASE_MODEL.distinct.each_batch(of: BATCH_SIZE, column: :project_id) do |batch, index|
+      project_ids = batch.pluck(:project_id)
+      migrate_in(index * DELAY_INTERVAL, MIGRATION_CLASS, project_ids)
+    end
+  end
+
+  def down; end
+
+  private
+
+  def run_migration?
+    Gitlab.ee? && table_exists?(:projects) && table_exists?(:vulnerabilities)
+  end
+end

db/schema_migrations/20200806100713

+fdcce45050f972d8edf2c645022f517ff6b9f4c76767e6cebe45a11fe34dd388
\ No newline at end of file

ee/changelogs/unreleased/227114_populate_resolved_on_default_branch_column_for_existing_vulnera.yml

+---
+title: Populate `resolved_on_default_branch` column for existing vulnerabilities
+merge_request: 38795
+author:
+type: added

ee/spec/lib/ee/gitlab/background_migration/populate_resolved_on_default_branch_column_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe ::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn do
+  let(:users) { table(:users) }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:pipelines) { table(:ci_pipelines) }
+  let(:vulnerabilities) { table(:vulnerabilities) }
+  let(:findings) { table(:vulnerability_occurrences) }
+  let(:builds) { table(:ci_builds) }
+  let(:scanners) { table(:vulnerability_scanners) }
+  let(:vulnerability_identifiers) { table(:vulnerability_identifiers) }
+
+  let(:namespace) { namespaces.create!(name: "foo", path: "bar") }
+
+  describe '#perform' do
+    let!(:project_1) { projects.create!(namespace_id: namespace.id) }
+    let!(:project_2) { projects.create!(namespace_id: namespace.id) }
+    let(:utility_class) { described_class::PopulateResolvedOnDefaultBranchColumnForProject }
+
+    subject(:populate_resolved_on_default_branch_column) { described_class.new.perform([project_1.id, project_2.id]) }
+
+    before do
+      allow(utility_class).to receive(:perform)
+    end
+
+    it 'calls `PopulateResolvedOnDefaultBranchColumnForProject.perform` for each project by given ids' do
+      populate_resolved_on_default_branch_column
+
+      expect(utility_class).to have_received(:perform).twice
+      expect(utility_class).to have_received(:perform).with(project_1.id)
+      expect(utility_class).to have_received(:perform).with(project_2.id)
+    end
+  end
+
+  describe EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::PopulateResolvedOnDefaultBranchColumnForProject do
+    describe '.perform' do
+      let(:project_id) { 1 }
+      let(:mock_utility_object) { instance_double(described_class, perform: true) }
+
+      subject(:populate_for_project) { described_class.perform(project_id) }
+
+      before do
+        allow(described_class).to receive(:new).and_return(mock_utility_object)
+      end
+
+      it 'instantiates the utility service object and calls #perform on it' do
+        populate_for_project
+
+        expect(described_class).to have_received(:new).with(project_id)
+        expect(mock_utility_object).to have_received(:perform)
+      end
+    end
+
+    describe '#perform' do
+      let(:user) { users.create!(name: 'John Doe', email: 'test@example.com', projects_limit: 5) }
+      let(:project) { projects.create!(namespace_id: namespace.id) }
+      let(:pipeline) { pipelines.create!(project_id: project.id, ref: 'master', sha: 'adf43c3a', status: 'success') }
+      let(:utility_object) { described_class.new(project.id) }
+      let(:scanner) { scanners.create!(project_id: project.id, external_id: 'bandit', name: 'Bandit') }
+      let(:artifact_model) { EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn::JobArtifact }
+      let(:artifact_fixture_path) { Rails.root.join('ee/spec/fixtures/security_reports/master/gl-sast-report.json') }
+      let(:sha_attribute) { Gitlab::Database::ShaAttribute.new }
+      let(:vulnerability_identifier) do
+        vulnerability_identifiers.create!(
+          project_id: project.id,
+          name: 'identifier',
+          fingerprint: sha_attribute.serialize('e6dd15eda2137be0034977a85b300a94a4f243a3'),
+          external_type: 'bar',
+          external_id: 'zoo')
+      end
+
+      let(:disappeared_vulnerability) do
+        vulnerabilities.create!(
+          project_id: project.id,
+          author_id: user.id,
+          title: 'Vulnerability',
+          severity: 5,
+          confidence: 5,
+          report_type: 5
+        )
+      end
+
+      let(:existing_vulnerability) do
+        vulnerabilities.create!(
+          project_id: project.id,
+          author_id: user.id,
+          title: 'Vulnerability',
+          severity: 5,
+          confidence: 5,
+          report_type: 5
+        )
+      end
+
+      subject(:populate_for_project) { utility_object.perform }
+
+      before do
+        build = builds.create!(commit_id: pipeline.id, retried: false, type: 'Ci::Build')
+        artifact = artifact_model.new(project_id: project.id, job_id: build.id, file_type: 5, file_format: 1)
+        artifact.file = fixture_file_upload(artifact_fixture_path, 'application/json')
+        artifact.save!
+
+        findings.create!(
+          project_id: project.id,
+          vulnerability_id: existing_vulnerability.id,
+          severity: 5,
+          confidence: 5,
+          report_type: 5,
+          scanner_id: scanner.id,
+          primary_identifier_id: vulnerability_identifier.id,
+          project_fingerprint: 'foo',
+          location_fingerprint: sha_attribute.serialize('d869ba3f0b3347eb2749135a437dc07c8ae0f420'),
+          uuid: SecureRandom.uuid,
+          name: 'Solar blast vulnerability',
+          metadata_version: '1',
+          raw_metadata: '')
+
+        allow(::Gitlab::CurrentSettings).to receive(:default_branch_name).and_return(:master)
+      end
+
+      it 'sets `resolved_on_default_branch` attribute of disappeared vulnerabilities' do
+        expect { populate_for_project }.to change { disappeared_vulnerability.reload[:resolved_on_default_branch] }.from(false).to(true)
+                                       .and not_change { existing_vulnerability.reload[:resolved_on_default_branch] }
+      end
+    end
+  end
+end

ee/spec/migrations/schedule_populate_resolved_on_default_branch_column_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+require Rails.root.join('db', 'post_migrate', '20200806100713_schedule_populate_resolved_on_default_branch_column.rb')
+
+RSpec.describe SchedulePopulateResolvedOnDefaultBranchColumn do
+  before do
+    allow_any_instance_of(Gitlab).to receive(:ee?).and_return(ee?)
+  end
+
+  around do |example|
+    Timecop.freeze { Sidekiq::Testing.fake! { example.run } }
+  end
+
+  context 'when the Gitlab instance is CE' do
+    let(:ee?) { false }
+
+    it 'does not run the migration' do
+      expect { migrate! }.not_to change { BackgroundMigrationWorker.jobs.size }
+    end
+  end
+
+  context 'when the Gitlab instance is EE' do
+    let(:ee?) { true }
+    let(:namespaces) { table(:namespaces) }
+    let(:projects) { table(:projects) }
+    let(:vulnerabilities) { table(:vulnerabilities) }
+    let(:users) { table(:users) }
+    let(:namespace) { namespaces.create!(name: "foo", path: "bar") }
+    let!(:project_1) { projects.create!(namespace_id: namespace.id) }
+    let!(:project_2) { projects.create!(namespace_id: namespace.id) }
+    let!(:project_3) { projects.create!(namespace_id: namespace.id) }
+    let(:user) { users.create!(name: 'John Doe', email: 'test@example.com', projects_limit: 1) }
+    let(:vulnerability_data) do
+      {
+        author_id: user.id,
+        title: 'Vulnerability',
+        severity: 5,
+        confidence: 5,
+        report_type: 5
+      }
+    end
+
+    before do
+      vulnerabilities.create!(**vulnerability_data, project_id: project_1.id)
+      vulnerabilities.create!(**vulnerability_data, project_id: project_2.id)
+
+      stub_const("#{described_class.name}::BATCH_SIZE", 1)
+    end
+
+    it 'schedules the background jobs', :aggregate_failures do
+      migrate!
+
+      expect(BackgroundMigrationWorker.jobs.size).to be(2)
+      expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(5.minutes, project_1.id)
+      expect(described_class::MIGRATION_CLASS).to be_scheduled_delayed_migration(10.minutes, project_2.id)
+    end
+  end
+end

lib/gitlab/background_migration/populate_resolved_on_default_branch_column.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop:disable Style/Documentation
+    class PopulateResolvedOnDefaultBranchColumn
+      def perform(*); end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn.prepend_if_ee('EE::Gitlab::BackgroundMigration::PopulateResolvedOnDefaultBranchColumn')