Merge branch '213571-migrate-lm-to-ls-artifacts' into 'master'

Migrate license_management artifacts to license_scanning

See merge request gitlab-org/gitlab!36817

changelogs/unreleased/213571-migrate-lm-to-ls-artifacts.yml

+---
+title: Migrate license_management artifacts to license_scanning type
+merge_request: 36817
+author:
+type: changed

db/post_migrate/20200808221641_add_index_for_license_compliance_artifacts.rb

+# frozen_string_literal: true
+
+class AddIndexForLicenseComplianceArtifacts < ActiveRecord::Migration[6.0]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+  INDEX_NAME = 'index_ci_job_artifacts_on_license_compliance_file_types'
+
+  disable_ddl_transaction!
+
+  def up
+    add_concurrent_index :ci_job_artifacts, [:job_id, :file_type], where: 'file_type = 10 OR file_type = 101', name: INDEX_NAME
+  end
+
+  def down
+    remove_concurrent_index_by_name :ci_job_artifacts, name: INDEX_NAME
+  end
+end

db/post_migrate/20200809221641_migrate_license_management_artifacts_to_license_scanning.rb

+# frozen_string_literal: true
+
+class MigrateLicenseManagementArtifactsToLicenseScanning < ActiveRecord::Migration[6.0]
+  DOWNTIME = false
+  LICENSE_MANAGEMENT_FILE_TYPE = 10
+  LICENSE_SCANNING_FILE_TYPE = 101
+
+  disable_ddl_transaction!
+
+  class JobArtifact < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_job_artifacts'
+  end
+
+  # We're updating file_type of ci artifacts from license_management to license_scanning
+  # But before that we need to delete "rogue" artifacts for CI builds that have associated with them
+  # both license_scanning and license_management artifacts. It's an edge case and usually, we don't have
+  # such builds in the database.
+  def up
+    return unless Gitlab.ee?
+
+    JobArtifact
+      .where("file_type = 10 OR file_type = 101")
+      .each_batch(column: :job_id, of: 1000) do |relation|
+      min, max = relation.pluck('MIN(job_id)', 'MAX(job_id)').flatten
+
+      ActiveRecord::Base.connection.execute <<~SQL
+        WITH ci_job_artifacts_with_row_number as (
+          SELECT job_id, id, ROW_NUMBER() OVER (PARTITION BY job_id ORDER BY id ASC) as row_number
+          FROM ci_job_artifacts
+          WHERE (file_type = #{LICENSE_SCANNING_FILE_TYPE} OR file_type = #{LICENSE_MANAGEMENT_FILE_TYPE})
+          AND job_id >= #{Integer(min)} AND job_id < #{Integer(max)}
+        )
+        DELETE FROM ci_job_artifacts
+        WHERE ci_job_artifacts.id IN (SELECT id from ci_job_artifacts_with_row_number WHERE ci_job_artifacts_with_row_number.row_number > 1)
+      SQL
+    end
+
+    JobArtifact.where(file_type: LICENSE_MANAGEMENT_FILE_TYPE).each_batch(of: 1000) do |relation|
+      relation.update_all(file_type: LICENSE_SCANNING_FILE_TYPE)
+    end
+  end
+
+  def down
+    # no-op
+    # we're deleting duplicating artifacts and updating file_type for license_management artifacts
+  end
+end

db/schema_migrations/20200808221641

+a24aa5052d37bff1bffc5df076d8422ea90f3781ae01ecf626bc59f3e299c90b
\ No newline at end of file

db/schema_migrations/20200809221641

+4e360aa1b375e391ec1202f1fe2eb26d64895faf326ec9c7a9b8d8351b6f4dc3
\ No newline at end of file

db/structure.sql

@@ -19202,6 +19202,8 @@ CREATE INDEX index_ci_job_artifacts_on_file_store ON public.ci_job_artifacts USI
 
 CREATE UNIQUE INDEX index_ci_job_artifacts_on_job_id_and_file_type ON public.ci_job_artifacts USING btree (job_id, file_type);
 
+CREATE INDEX index_ci_job_artifacts_on_license_compliance_file_types ON public.ci_job_artifacts USING btree (job_id, file_type) WHERE ((file_type = 10) OR (file_type = 101));
+
 CREATE INDEX index_ci_job_artifacts_on_project_id ON public.ci_job_artifacts USING btree (project_id);
 
 CREATE INDEX index_ci_job_artifacts_on_project_id_for_security_reports ON public.ci_job_artifacts USING btree (project_id) WHERE (file_type = ANY (ARRAY[5, 6, 7, 8]));

ee/spec/migrations/migrate_license_management_artifacts_to_license_scanning_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require Rails.root.join('db', 'post_migrate', '20200809221641_migrate_license_management_artifacts_to_license_scanning.rb')
+
+RSpec.describe MigrateLicenseManagementArtifactsToLicenseScanning, :migration, :sidekiq do
+  let(:namespaces) { table(:namespaces) }
+  let(:projects) { table(:projects) }
+  let(:job_artifacts) { table(:ci_job_artifacts) }
+  let(:builds) { table(:ci_builds) }
+  let(:license_management_type) {  Ci::JobArtifact.file_types[:license_management] }
+  let(:license_scanning_type) { Ci::JobArtifact.file_types[:license_scanning] }
+
+  before do
+    namespaces.create!(id: 1, name: 'tanuki', path: 'tanuki')
+    projects.create!(id: 42, name: 'tanuki', path: 'tanuki', namespace_id: 1)
+    builds.create!(id: 1)
+    builds.create!(id: 2)
+    builds.create!(id: 3)
+    job_artifacts.create!(project_id: 42, job_id: 1, file_type: 10)
+    job_artifacts.create!(project_id: 42, job_id: 2, file_type: 9)
+    job_artifacts.create!(project_id: 42, job_id: 2, file_type: 10)
+  end
+
+  context 'with two types of the report' do
+    before do
+      job_artifacts.create!(project_id: 42, job_id: 1, file_type: 101)
+    end
+
+    it 'leaves only one artifact' do
+      migrate!
+
+      expect(job_artifacts.where(file_type: 10).count).to eq 0
+      expect(job_artifacts.where(file_type: 101).count).to eq 2
+      expect(job_artifacts.where(file_type: 9).count).to eq 1
+    end
+  end
+
+  context 'with only license_management report' do
+    it 'changes license_management to license_scanning' do
+      migrate!
+
+      expect(job_artifacts.where(file_type: 10).count).to eq 0
+      expect(job_artifacts.where(file_type: 101).count).to eq 2
+      expect(job_artifacts.where(file_type: 9).count).to eq 1
+    end
+  end
+
+  context 'with FOSS version of GitLab' do
+    before do
+      allow(Gitlab).to receive(:ee?).and_return(false)
+    end
+
+    it 'skips this migration' do
+      migrate!
+
+      expect(job_artifacts.where(file_type: 10).count).to eq 2
+      expect(job_artifacts.where(file_type: 101).count).to eq 0
+    end
+  end
+end