Merge branch 'ab/batched-migration-metrics' into 'master'

Report migrated tuples count in total

See merge request gitlab-org/gitlab!60264

lib/gitlab/database/background_migration/batched_migration.rb

@@ -64,6 +64,10 @@ module Gitlab
           write_attribute(:batch_class_name, class_name.demodulize)
         end
 
+        def migrated_tuple_count
+          batched_jobs.succeeded.sum(:batch_size)
+        end
+
         def prometheus_labels
           @prometheus_labels ||= {
             migration_id: id,

lib/gitlab/database/background_migration/batched_migration_wrapper.rb

@@ -65,6 +65,7 @@ module Gitlab
           metric_for(:gauge_interval).set(base_labels, tracking_record.batched_migration.interval)
           metric_for(:gauge_job_duration).set(base_labels, (tracking_record.finished_at - tracking_record.started_at).to_i)
           metric_for(:counter_updated_tuples).increment(base_labels, tracking_record.batch_size)
+          metric_for(:gauge_migrated_tuples).set(base_labels, tracking_record.batched_migration.migrated_tuple_count)
           metric_for(:gauge_total_tuple_count).set(base_labels, tracking_record.batched_migration.total_tuple_count)
 
           if metrics = tracking_record.metrics
@@ -106,6 +107,10 @@ module Gitlab
                 :batched_migration_job_updated_tuples_total,
                 'Number of tuples updated by batched migration job'
               ),
+              gauge_migrated_tuples: Gitlab::Metrics.gauge(
+                :batched_migration_migrated_tuples_total,
+                'Total number of tuples migrated by a batched migration'
+              ),
               histogram_timings: Gitlab::Metrics.histogram(
                 :batched_migration_job_query_duration_seconds,
                 'Query timings for a batched migration job',

spec/lib/gitlab/database/background_migration/batched_migration_spec.rb

@@ -204,6 +204,22 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m
     it_behaves_like 'an attr_writer that demodulizes assigned class names', :batch_class_name
   end
 
+  describe '#migrated_tuple_count' do
+    subject { batched_migration.migrated_tuple_count }
+
+    let(:batched_migration) { create(:batched_background_migration) }
+
+    before do
+      create_list(:batched_background_migration_job, 5, status: :succeeded, batch_size: 1_000, batched_migration: batched_migration)
+      create_list(:batched_background_migration_job, 1, status: :running, batch_size: 1_000, batched_migration: batched_migration)
+      create_list(:batched_background_migration_job, 1, status: :failed, batch_size: 1_000, batched_migration: batched_migration)
+    end
+
+    it 'sums the batch_size of succeeded jobs' do
+      expect(subject).to eq(5_000)
+    end
+  end
+
   describe '#prometheus_labels' do
     let(:batched_migration) { create(:batched_background_migration, job_class_name: 'TestMigration', table_name: 'foo', column_name: 'bar') }
 

spec/lib/gitlab/database/background_migration/batched_migration_wrapper_spec.rb

@@ -80,6 +80,14 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigrationWrapper, '
       subject
     end
 
+    it 'reports migrated tuples' do
+      count = double
+      expect(job_record.batched_migration).to receive(:migrated_tuple_count).and_return(count)
+      expect(described_class.metrics[:gauge_migrated_tuples]).to receive(:set).with(labels, count)
+
+      subject
+    end
+
     it 'reports summary of query timings' do
       metrics = { 'timings' => { 'update_all' => [1, 2, 3, 4, 5] } }