Merge branch 'eb-backfill-ci-feature-usage-for-coverage' into 'master'

Backfill projects with CI coverage usage

See merge request gitlab-org/gitlab!69115

db/post_migrate/20210818185845_backfill_projects_with_coverage.rb

+# frozen_string_literal: true
+
+class BackfillProjectsWithCoverage < ActiveRecord::Migration[6.1]
+  include Gitlab::Database::MigrationHelpers
+
+  MIGRATION = 'BackfillProjectsWithCoverage'
+  DELAY_INTERVAL = 2.minutes
+  BATCH_SIZE = 10_000
+  SUB_BATCH_SIZE = 1_000
+
+  disable_ddl_transaction!
+
+  class CiDailyBuildGroupReportResult < ActiveRecord::Base
+    include EachBatch
+
+    self.table_name = 'ci_daily_build_group_report_results'
+  end
+
+  def up
+    queue_background_migration_jobs_by_range_at_intervals(
+      CiDailyBuildGroupReportResult,
+      MIGRATION,
+      DELAY_INTERVAL,
+      batch_size: BATCH_SIZE,
+      other_job_arguments: [SUB_BATCH_SIZE]
+    )
+  end
+
+  def down
+    # noop
+  end
+end

db/schema_migrations/20210818185845

+dc8ca347fb0c87e1a66389fd9f37fa9702f8bd53237ada40192bb0a875dbe940
\ No newline at end of file

lib/gitlab/background_migration/backfill_projects_with_coverage.rb

+# frozen_string_literal: true
+
+module Gitlab
+  module BackgroundMigration
+    # Backfill project_ci_feature_usages for a range of projects with coverage
+    class BackfillProjectsWithCoverage
+      class ProjectCiFeatureUsage < ActiveRecord::Base # rubocop:disable Style/Documentation
+        self.table_name = 'project_ci_feature_usages'
+      end
+
+      COVERAGE_ENUM_VALUE = 1
+      INSERT_DELAY_SECONDS = 0.1
+
+      def perform(start_id, end_id, sub_batch_size)
+        report_results = ActiveRecord::Base.connection.execute <<~SQL
+          SELECT DISTINCT project_id, default_branch
+          FROM ci_daily_build_group_report_results
+          WHERE id BETWEEN #{start_id} AND #{end_id}
+        SQL
+
+        report_results.to_a.in_groups_of(sub_batch_size, false) do |batch|
+          ProjectCiFeatureUsage.insert_all(build_values(batch))
+
+          sleep INSERT_DELAY_SECONDS
+        end
+      end
+
+      private
+
+      def build_values(batch)
+        batch.map do |data|
+          {
+            project_id: data['project_id'],
+            feature: COVERAGE_ENUM_VALUE,
+            default_branch: data['default_branch']
+          }
+        end
+      end
+    end
+  end
+end

spec/lib/gitlab/background_migration/backfill_projects_with_coverage_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe Gitlab::BackgroundMigration::BackfillProjectsWithCoverage, schema: 20210818185845 do
+  let(:projects) { table(:projects) }
+  let(:project_ci_feature_usages) { table(:project_ci_feature_usages) }
+  let(:ci_pipelines) { table(:ci_pipelines) }
+  let(:ci_daily_build_group_report_results) { table(:ci_daily_build_group_report_results) }
+  let(:group) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:project_1) { projects.create!(namespace_id: group.id) }
+  let(:project_2) { projects.create!(namespace_id: group.id) }
+  let(:pipeline_1) { ci_pipelines.create!(project_id: project_1.id, source: 13) }
+  let(:pipeline_2) { ci_pipelines.create!(project_id: project_1.id, source: 13) }
+  let(:pipeline_3) { ci_pipelines.create!(project_id: project_2.id, source: 13) }
+  let(:pipeline_4) { ci_pipelines.create!(project_id: project_2.id, source: 13) }
+
+  subject { described_class.new }
+
+  describe '#perform' do
+    before do
+      ci_daily_build_group_report_results.create!(
+        id: 1,
+        project_id: project_1.id,
+        date: 4.days.ago,
+        last_pipeline_id: pipeline_1.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 2,
+        project_id: project_1.id,
+        date: 3.days.ago,
+        last_pipeline_id: pipeline_2.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 3,
+        project_id: project_2.id,
+        date: 2.days.ago,
+        last_pipeline_id: pipeline_3.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 4,
+        project_id: project_2.id,
+        date: 1.day.ago,
+        last_pipeline_id: pipeline_4.id,
+        ref_path: 'test_branch',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: false,
+        group_id: group.id
+      )
+
+      stub_const("#{described_class}::INSERT_DELAY_SECONDS", 0)
+    end
+
+    it 'creates entries per project and default_branch combination in the given range', :aggregate_failures do
+      subject.perform(1, 4, 2)
+
+      entries = project_ci_feature_usages.order('project_id ASC, default_branch DESC')
+
+      expect(entries.count).to eq(3)
+      expect(entries[0]).to have_attributes(project_id: project_1.id, feature: 1, default_branch: true)
+      expect(entries[1]).to have_attributes(project_id: project_2.id, feature: 1, default_branch: true)
+      expect(entries[2]).to have_attributes(project_id: project_2.id, feature: 1, default_branch: false)
+    end
+
+    context 'when an entry for the project and default branch combination already exists' do
+      before do
+        subject.perform(1, 4, 2)
+      end
+
+      it 'does not create a new entry' do
+        expect { subject.perform(1, 4, 2) }.not_to change { project_ci_feature_usages.count }
+      end
+    end
+  end
+end

spec/migrations/20210818185845_backfill_projects_with_coverage_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+require_migration!('backfill_projects_with_coverage')
+
+RSpec.describe BackfillProjectsWithCoverage do
+  let(:projects) { table(:projects) }
+  let(:ci_pipelines) { table(:ci_pipelines) }
+  let(:ci_daily_build_group_report_results) { table(:ci_daily_build_group_report_results) }
+  let(:group) { table(:namespaces).create!(name: 'user', path: 'user') }
+  let(:project_1) { projects.create!(namespace_id: group.id) }
+  let(:project_2) { projects.create!(namespace_id: group.id) }
+  let(:pipeline_1) { ci_pipelines.create!(project_id: project_1.id) }
+  let(:pipeline_2) { ci_pipelines.create!(project_id: project_2.id) }
+  let(:pipeline_3) { ci_pipelines.create!(project_id: project_2.id) }
+
+  describe '#up' do
+    before do
+      stub_const("#{described_class}::BATCH_SIZE", 2)
+      stub_const("#{described_class}::SUB_BATCH_SIZE", 1)
+
+      ci_daily_build_group_report_results.create!(
+        id: 1,
+        project_id: project_1.id,
+        date: 3.days.ago,
+        last_pipeline_id: pipeline_1.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 2,
+        project_id: project_2.id,
+        date: 2.days.ago,
+        last_pipeline_id: pipeline_2.id,
+        ref_path: 'main',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: true,
+        group_id: group.id
+      )
+
+      ci_daily_build_group_report_results.create!(
+        id: 3,
+        project_id: project_2.id,
+        date: 1.day.ago,
+        last_pipeline_id: pipeline_3.id,
+        ref_path: 'test_branch',
+        group_name: 'rspec',
+        data: { coverage: 95.0 },
+        default_branch: false,
+        group_id: group.id
+      )
+    end
+
+    it 'schedules BackfillProjectsWithCoverage background jobs', :aggregate_failures do
+      Sidekiq::Testing.fake! do
+        freeze_time do
+          migrate!
+
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(2.minutes, 1, 2, 1)
+          expect(described_class::MIGRATION).to be_scheduled_delayed_migration(4.minutes, 3, 3, 1)
+          expect(BackgroundMigrationWorker.jobs.size).to eq(2)
+        end
+      end
+    end
+  end
+end