Validate batched migration jobs

- Before moving to state 'finished' validates if jobs are succeeded

Related to https://gitlab.com/gitlab-org/gitlab/-/issues/342352

Changelog: added

lib/gitlab/database/background_migration/batched_job.rb

@@ -21,6 +21,7 @@ module Gitlab
 
           from_union([failed_jobs, self.stuck])
         }
+        scope :except_succeeded, -> { where(status: self.statuses.except(:succeeded).values) }
 
         enum status: {
           pending: 0,

lib/gitlab/database/background_migration/batched_migration.rb

@@ -18,6 +18,8 @@ module Gitlab
           scope: [:job_class_name, :table_name, :column_name]
         }
 
+        validate :validate_batched_jobs_status, if: -> { status_changed? && finished? }
+
         scope :queue_order, -> { order(id: :asc) }
         scope :queued, -> { where(status: [:active, :paused]) }
         scope :for_configuration, ->(job_class_name, table_name, column_name, job_arguments) do
@@ -133,6 +135,12 @@ module Gitlab
         def optimize!
           BatchOptimizer.new(self).optimize!
         end
+
+        private
+
+        def validate_batched_jobs_status
+          errors.add(:batched_jobs, 'jobs need to be succeeded') if batched_jobs.except_succeeded.exists?
+        end
       end
     end
   end

spec/lib/gitlab/database/background_migration/batched_job_spec.rb

@@ -17,15 +17,19 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedJob, type: :model d
     let_it_be(:stuck_job) { create(:batched_background_migration_job, status: :pending, updated_at: fixed_time - described_class::STUCK_JOBS_TIMEOUT) }
     let_it_be(:failed_job) { create(:batched_background_migration_job, status: :failed, attempts: 1) }
 
-    before_all do
-      create(:batched_background_migration_job, status: :failed, attempts: described_class::MAX_ATTEMPTS)
-      create(:batched_background_migration_job, status: :succeeded)
-    end
+    let!(:max_attempts_failed_job) { create(:batched_background_migration_job, status: :failed, attempts: described_class::MAX_ATTEMPTS) }
+    let!(:succeeded_job) { create(:batched_background_migration_job, status: :succeeded) }
 
     before do
       travel_to fixed_time
     end
 
+    describe '.except_succeeded' do
+      it 'returns not succeeded jobs' do
+        expect(described_class.except_succeeded).to contain_exactly(pending_job, running_job, stuck_job, failed_job, max_attempts_failed_job)
+      end
+    end
+
     describe '.active' do
       it 'returns active jobs' do
         expect(described_class.active).to contain_exactly(pending_job, running_job, stuck_job)

spec/lib/gitlab/database/background_migration/batched_migration_spec.rb

@@ -23,6 +23,28 @@ RSpec.describe Gitlab::Database::BackgroundMigration::BatchedMigration, type: :m
     subject { build(:batched_background_migration) }
 
     it { is_expected.to validate_uniqueness_of(:job_arguments).scoped_to(:job_class_name, :table_name, :column_name) }
+
+    context 'when there are failed jobs' do
+      let(:batched_migration) { create(:batched_background_migration, status: :active, total_tuple_count: 100) }
+      let!(:batched_job) { create(:batched_background_migration_job, batched_migration: batched_migration, status: :failed) }
+
+      it 'raises an exception' do
+        expect { batched_migration.finished! }.to raise_error(ActiveRecord::RecordInvalid)
+
+        expect(batched_migration.reload.status).to eql 'active'
+      end
+    end
+
+    context 'when the jobs are completed' do
+      let(:batched_migration) { create(:batched_background_migration, status: :active, total_tuple_count: 100) }
+      let!(:batched_job) { create(:batched_background_migration_job, batched_migration: batched_migration, status: :succeeded) }
+
+      it 'finishes the migration' do
+        batched_migration.finished!
+
+        expect(batched_migration.status).to eql 'finished'
+      end
+    end
   end
 
   describe '.queue_order' do