Backfill projects with CI coverage usage

This backfills the projects_ci_feature_usages table
with projects that uses the CI coverage feature through
a background migration.

Changelog: added

db/post_migrate/20210818185845_backfill_projects_with_coverage.rb

+# frozen_string_literal: true
+
+class BackfillProjectsWithCoverage < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  DOWNTIME = false
+
+  MIGRATION = 'BackfillProjectsWithCoverage'
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 10_000
+
+  disable_ddl_transaction!
+
+  class CiDailyBuildGroupReportResult < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_daily_build_group_report_results'
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      CiDailyBuildGroupReportResult,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE
+    )
+  end
+
+  def down
+    # noop
+  end
+end

db/schema_migrations/20210818185845

+dc8ca347fb0c87e1a66389fd9f37fa9702f8bd53237ada40192bb0a875dbe940
\ No newline at end of file

lib/gitlab/background_migration/backfill_projects_with_coverage.rb

+# frozen_string_literal: true
+
+# Backfill project_ci_feature_usages for a range of projects with coverage
+class Gitlab::BackgroundMigration::BackfillProjectsWithCoverage
+  def perform(start_id, end_id)
+    ActiveRecord::Base.connection.execute <<~SQL
+      INSERT INTO project_ci_feature_usages (project_id, feature, default_branch)
+        SELECT DISTINCT project_id, 1 as feature, default_branch
+        FROM ci_daily_build_group_report_results
+        WHERE id BETWEEN #{start_id} AND #{end_id}
+      ON CONFLICT (project_id, feature, default_branch) DO NOTHING;
+    SQL
+  end
+end

spec/lib/gitlab/background_migration/backfill_projects_with_coverage_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillProjectsWithCoverage, schema: 20210818185845 do
+  let(:projects) { table(:projects) }
+  let(:project_ci_feature_usages) { table(:project_ci_feature_usages) }
+  let(:ci_pipelines) { table(:ci_pipelines) }
+  let(:ci_daily_build_group_report_results) { table(:ci_daily_build_group_report_results) }
+  let(:group) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:project_1) { projects.create!(namespace_id: group.id) }
+  let(:project_2) { projects.create!(namespace_id: group.id) }
+  let(:pipeline_1) { ci_pipelines.create!(project_id: project_1.id, source: 13) }
+  let(:pipeline_2) { ci_pipelines.create!(project_id: project_1.id, source: 13) }
+  let(:pipeline_3) { ci_pipelines.create!(project_id: project_2.id, source: 13) }
+  let(:pipeline_4) { ci_pipelines.create!(project_id: project_2.id, source: 13) }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    before do
+      ci_daily_build_group_report_results.create!(
+        id: 1,
+        project_id: project_1.id,
+        date: 4.days.ago,
+        last_pipeline_id: pipeline_1.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 2,
+        project_id: project_1.id,
+        date: 3.days.ago,
+        last_pipeline_id: pipeline_2.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 3,
+        project_id: project_2.id,
+        date: 2.days.ago,
+        last_pipeline_id: pipeline_3.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 4,
+        project_id: project_2.id,
+        date: 1.day.ago,
+        last_pipeline_id: pipeline_4.id,
+        ref_path: 'test_branch',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: false,
+        group_id: group.id
+      )
+    end
+
+    it 'creates entries per project and default_branch combination in the given range' do
+      subject.perform(1, 4)
+
+      entries = project_ci_feature_usages.order('project_id ASC, default_branch DESC')
+
+      expect(entries.count).to eq(3)
+      expect(entries[0]).to have_attributes(project_id: project_1.id, feature: 1, default_branch: true)
+      expect(entries[1]).to have_attributes(project_id: project_2.id, feature: 1, default_branch: true)
+      expect(entries[2]).to have_attributes(project_id: project_2.id, feature: 1, default_branch: false)
+    end
+
+    context 'when an entry for the project and default branch combination already exists' do
+      before do
+        subject.perform(1, 4)
+      end
+
+      it 'does not create a new entry' do
+        expect { subject.perform(1, 4) }.not_to change { project_ci_feature_usages.count }
+        expect(project_ci_feature_usages.count).to eq(3)
+      end
+    end
+  end
+end

spec/migrations/20210818185845_backfill_projects_with_coverage_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!('backfill_projects_with_coverage')
+
+RSpec.describe BackfillProjectsWithCoverage do
+  let(:projects) { table(:projects) }
+  let(:ci_pipelines) { table(:ci_pipelines) }
+  let(:ci_daily_build_group_report_results) { table(:ci_daily_build_group_report_results) }
+  let(:group) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:project_1) { projects.create!(namespace_id: group.id) }
+  let(:project_2) { projects.create!(namespace_id: group.id) }
+  let(:pipeline_1) { ci_pipelines.create!(project_id: project_1.id) }
+  let(:pipeline_2) { ci_pipelines.create!(project_id: project_2.id) }
+  let(:pipeline_3) { ci_pipelines.create!(project_id: project_2.id) }
+
+  describe '#up' do
+    before do
+      stub_const("#{described_class}::BATCH_SIZE", 2)
+
+      ci_daily_build_group_report_results.create!(
+        id: 1,
+        project_id: project_1.id,
+        date: 3.days.ago,
+        last_pipeline_id: pipeline_1.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 2,
+        project_id: project_2.id,
+        date: 2.days.ago,
+        last_pipeline_id: pipeline_2.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 3,
+        project_id: project_2.id,
+        date: 1.day.ago,
+        last_pipeline_id: pipeline_3.id,
+        ref_path: 'test_branch',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: false,
+        group_id: group.id
+      )
+    end
+
+    it 'schedules BackfillProjectsWithCoverage background jobs', :aggregate_failures do
+      Sidekiq::Testing.fake! do
+        freeze_time do
+          migrate!
+
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, 1, 2)
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, 3, 3)
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        end
+      end
+    end
+  end
+end