Merge branch 'migrate-security-scans' into 'master'

Migrate Secure Job Artifacts to Security Scans

Closes #10250

See merge request gitlab-org/gitlab!24125

changelogs/unreleased/migrate-security-scans.yml

+---
+title: Schedule worker to migrate security job artifacts to security scans
+merge_request: 24125
+author:
+type: other

db/post_migrate/20200217223651_add_index_to_job_artifact_secure_reports.rb

+# frozen_string_literal: true
+
+class AddIndexToJobArtifactSecureReports < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'job_artifacts_secure_reports_temp_index'
+  PARTIAL_FILTER = "file_type BETWEEN 5 AND 8"
+
+  disable_ddl_transaction!
+
+  def up
+    # This is a temporary index used for the migration of Security Reports to Security Scans
+    add_concurrent_index(:ci_job_artifacts,
+                         [:id, :file_type, :job_id, :created_at, :updated_at],
+                         name: INDEX_NAME,
+                         where: PARTIAL_FILTER)
+  end
+
+  def down
+    remove_concurrent_index(:ci_job_artifacts,
+                            [:id, :file_type, :job_id, :created_at, :updated_at])
+  end
+end

db/post_migrate/20200217225719_schedule_migrate_security_scans.rb

+# frozen_string_literal: true
+
+class ScheduleMigrateSecurityScans < ActiveRecord::Migration[5.2]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INTERVAL = 2.minutes.to_i
+  BATCH_SIZE = 10_000
+  MIGRATION = 'MigrateSecurityScans'.freeze
+
+  disable_ddl_transaction!
+
+  class JobArtifact < ActiveRecord::Base
+    include ::EachBatch
+
+    self.table_name = 'ci_job_artifacts'
+
+    scope :security_reports, -> { where('file_type BETWEEN 5 and 8') }
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(JobArtifact.security_reports,
+                                                          MIGRATION,
+                                                          INTERVAL,
+                                                          batch_size: BATCH_SIZE)
+  end
+
+  def down
+    # intentionally blank
+  end
+end

db/schema.rb

@@ -766,6 +766,7 @@ ActiveRecord::Schema.define(version: 2020_02_26_162723) do
     t.integer "file_location", limit: 2
     t.index ["expire_at", "job_id"], name: "index_ci_job_artifacts_on_expire_at_and_job_id"
     t.index ["file_store"], name: "index_ci_job_artifacts_on_file_store"
+    t.index ["id", "file_type", "job_id", "created_at", "updated_at"], name: "job_artifacts_secure_reports_temp_index", where: "((file_type >= 5) AND (file_type <= 8))"
     t.index ["job_id", "file_type"], name: "index_ci_job_artifacts_on_job_id_and_file_type", unique: true
     t.index ["project_id"], name: "index_ci_job_artifacts_on_project_id"
     t.index ["project_id"], name: "index_ci_job_artifacts_on_project_id_for_security_reports", where: "(file_type = ANY (ARRAY[5, 6, 7, 8]))"

ee/lib/ee/gitlab/background_migration/migrate_security_scans.rb

+# frozen_string_literal: true
+
+# rubocop: disable Gitlab/ModuleWithInstanceVariables
+module EE
+  module Gitlab
+    module BackgroundMigration
+      module MigrateSecurityScans
+        extend ::Gitlab::Utils::Override
+
+        override :perform
+        def perform(start_id, stop_id)
+          execute <<~SQL
+            INSERT INTO security_scans (created_at, updated_at, build_id, scan_type)
+            SELECT ci_job_artifacts.created_at, ci_job_artifacts.updated_at, ci_job_artifacts.job_id, ci_job_artifacts.file_type - 4
+            FROM ci_job_artifacts
+            WHERE ci_job_artifacts.id BETWEEN #{start_id} AND #{stop_id}
+            AND ci_job_artifacts.file_type BETWEEN 5 and 8
+            ON CONFLICT (build_id, scan_type) DO NOTHING;
+          SQL
+        end
+
+        def execute(sql)
+          @connection ||= ::ActiveRecord::Base.connection
+          @connection.execute(sql)
+        end
+      end
+    end
+  end
+end

ee/spec/lib/ee/gitlab/background_migration/migrate_security_scans_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+# rubocop: disable RSpec/FactoriesInMigrationSpecs
+describe Gitlab::BackgroundMigration::MigrateSecurityScans, :migration, schema: 20200220180944 do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:builds) { table(:ci_builds) }
+  let(:job_artifacts) { table(:ci_job_artifacts) }
+  let(:security_scans) { table(:security_scans) }
+
+  let(:namespace) { namespaces.create!(name: "foo", path: "bar") }
+  let(:project) { projects.create!(namespace_id: namespace.id) }
+  let(:build) { builds.create! }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    context 'when job artifacts and builds are present' do
+      using RSpec::Parameterized::TableSyntax
+
+      where(:scan_type_name, :report_type, :scan_type_number) do
+        :sast | 5 | 1
+        :dependency_scanning | 6 | 2
+        :container_scanning | 7 | 3
+        :dast | 8 | 4
+      end
+
+      with_them do
+        let!(:job_artifact) do
+          job_artifacts.create!(
+            created_at: 10.minutes.ago,
+            updated_at: 9.minutes.ago,
+            project_id: project.id,
+            job_id: build.id,
+            file_type: report_type
+          )
+        end
+
+        it 'creates a new security scan' do
+          subject.perform(job_artifact.id, job_artifact.id)
+
+          scan = Security::Scan.first
+          expect(scan.build_id).to eq(build.id)
+          expect(scan.scan_type).to eq(scan_type_name.to_s)
+          expect(scan.created_at.to_s).to eq(job_artifact.created_at.to_s)
+          expect(scan.updated_at.to_s).to eq(job_artifact.updated_at.to_s)
+        end
+      end
+    end
+
+    context 'job artifacts are not found' do
+      it 'security scans are not created' do
+        subject.perform(1, 2)
+
+        expect(Security::Scan.count).to eq(0)
+      end
+    end
+  end
+
+  context 'security scan has already been saved' do
+    let!(:job_artifact) { job_artifacts.create!(project_id: project.id, job_id: build.id, file_type: 5) }
+
+    before do
+      security_scans.create!(build_id: build.id, scan_type: 1)
+    end
+
+    it 'does not save a new security scan' do
+      subject.perform(job_artifact.id, job_artifact.id)
+
+      expect(Security::Scan.count).to eq(1)
+    end
+  end
+
+  context 'job artifacts are not security job artifacts' do
+    let!(:job_artifact) { job_artifacts.create!(project_id: project.id, job_id: build.id, file_type: 1) }
+
+    it 'does not save a new security scan' do
+      subject.perform(job_artifact.id, job_artifact.id)
+
+      expect(Security::Scan.count).to eq(0)
+    end
+  end
+end

lib/gitlab/background_migration/migrate_security_scans.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # rubocop: disable Style/Documentation
+    class MigrateSecurityScans
+      def perform(start_id, stop_id)
+      end
+    end
+  end
+end
+
+Gitlab::BackgroundMigration::MigrateSecurityScans.prepend_if_ee('EE::Gitlab::BackgroundMigration::MigrateSecurityScans')

spec/migrations/schedule_migrate_security_scans_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200217225719_schedule_migrate_security_scans.rb')
+
+# rubocop: disable RSpec/FactoriesInMigrationSpecs
+describe ScheduleMigrateSecurityScans, :migration, :sidekiq do
+  let(:migration) { described_class.new }
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:builds) { table(:ci_builds) }
+  let(:job_artifacts) { table(:ci_job_artifacts) }
+
+  let(:namespace) { namespaces.create!(name: "foo", path: "bar") }
+  let(:project) { projects.create!(namespace_id: namespace.id) }
+  let(:build) { builds.create! }
+
+  before do
+    stub_const("#{described_class.name}::BATCH_SIZE", 1)
+    stub_const("#{described_class.name}::INTERVAL", 5.minutes.to_i)
+  end
+
+  context 'no security job artifacts' do
+    before do
+      table(:ci_job_artifacts)
+    end
+
+    it 'does not schedule migration' do
+      Sidekiq::Testing.fake! do
+        migrate!
+
+        expect(BackgroundMigrationWorker.jobs).to be_empty
+      end
+    end
+  end
+
+  context 'has security job artifacts' do
+    let!(:job_artifact_1) { job_artifacts.create!(project_id: project.id, job_id: build.id, file_type: 5) }
+    let!(:job_artifact_2) { job_artifacts.create!(project_id: project.id, job_id: build.id, file_type: 8) }
+
+    it 'schedules migration of security scans' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migration.up
+
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(5.minutes, job_artifact_1.id, job_artifact_1.id)
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(10.minutes, job_artifact_2.id, job_artifact_2.id)
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        end
+      end
+    end
+  end
+
+  context 'has non-security job artifacts' do
+    let!(:job_artifact_1) { job_artifacts.create!(project_id: project.id, job_id: build.id, file_type: 4) }
+    let!(:job_artifact_2) { job_artifacts.create!(project_id: project.id, job_id: build.id, file_type: 9) }
+
+    it 'schedules migration of security scans' do
+      Sidekiq::Testing.fake! do
+        Timecop.freeze do
+          migration.up
+
+          expect(BackgroundMigrationWorker.jobs).to be_empty
+        end
+      end
+    end
+  end
+end