Merge branch 'use-ordered-stages-for-atomic-processing' into 'master'

Use ordered stages for atomic processing

See merge request gitlab-org/gitlab!30487

app/controllers/projects/pipelines_controller.rb

@@ -70,6 +70,8 @@ class Projects::PipelinesController < Projects::ApplicationController
   end
 
   def show
+    Gitlab::QueryLimiting.whitelist('https://gitlab.com/gitlab-org/gitlab/-/issues/26657')
+
     respond_to do |format|
       format.html
       format.json do

app/models/ci/pipeline.rb

@@ -393,16 +393,18 @@ module Ci
       false
     end
 
-    ##
-    # TODO We do not completely switch to persisted stages because of
-    # race conditions with setting statuses gitlab-foss#23257.
-    #
     def ordered_stages
-      return legacy_stages unless complete?
-
-      if Feature.enabled?('ci_pipeline_persisted_stages', default_enabled: true)
+      if Feature.enabled?(:ci_atomic_processing, project, default_enabled: false)
+        # The `Ci::Stage` contains all up-to date data
+        # as atomic processing updates all data in-bulk
+        stages
+      elsif Feature.enabled?(:ci_pipeline_persisted_stages, default_enabled: true) && complete?
+        # The `Ci::Stage` contains up-to date data only for `completed` pipelines
+        # this is due to asynchronous processing of pipeline, and stages possibly
+        # not updated inline with processing of pipeline
         stages
       else
+        # In other cases, we need to calculate stages dynamically
         legacy_stages
       end
     end

spec/features/projects/pipelines/pipeline_spec.rb

@@ -19,32 +19,32 @@ describe 'Pipeline', :js do
   shared_context 'pipeline builds' do
     let!(:build_passed) do
       create(:ci_build, :success,
-             pipeline: pipeline, stage: 'build', name: 'build')
+             pipeline: pipeline, stage: 'build', stage_idx: 0, name: 'build')
     end
 
     let!(:build_failed) do
       create(:ci_build, :failed,
-             pipeline: pipeline, stage: 'test', name: 'test')
+             pipeline: pipeline, stage: 'test', stage_idx: 1, name: 'test')
     end
 
     let!(:build_preparing) do
       create(:ci_build, :preparing,
-             pipeline: pipeline, stage: 'deploy', name: 'prepare')
+             pipeline: pipeline, stage: 'deploy', stage_idx: 2, name: 'prepare')
     end
 
     let!(:build_running) do
       create(:ci_build, :running,
-             pipeline: pipeline, stage: 'deploy', name: 'deploy')
+             pipeline: pipeline, stage: 'deploy', stage_idx: 3, name: 'deploy')
     end
 
     let!(:build_manual) do
       create(:ci_build, :manual,
-             pipeline: pipeline, stage: 'deploy', name: 'manual-build')
+             pipeline: pipeline, stage: 'deploy', stage_idx: 3, name: 'manual-build')
     end
 
     let!(:build_scheduled) do
       create(:ci_build, :scheduled,
-             pipeline: pipeline, stage: 'deploy', name: 'delayed-job')
+             pipeline: pipeline, stage: 'deploy', stage_idx: 3, name: 'delayed-job')
     end
 
     let!(:build_external) do
@@ -307,9 +307,12 @@ describe 'Pipeline', :js do
 
     context 'when the pipeline has manual stage' do
       before do
-        create(:ci_build, :manual, pipeline: pipeline, stage: 'publish', name: 'CentOS')
-        create(:ci_build, :manual, pipeline: pipeline, stage: 'publish', name: 'Debian')
-        create(:ci_build, :manual, pipeline: pipeline, stage: 'publish', name: 'OpenSUDE')
+        create(:ci_build, :manual, pipeline: pipeline, stage_idx: 10, stage: 'publish', name: 'CentOS')
+        create(:ci_build, :manual, pipeline: pipeline, stage_idx: 10, stage: 'publish', name: 'Debian')
+        create(:ci_build, :manual, pipeline: pipeline, stage_idx: 10, stage: 'publish', name: 'OpenSUDE')
+
+        # force to update stages statuses
+        Ci::ProcessPipelineService.new(pipeline).execute
 
         visit_pipeline
       end

spec/models/ci/pipeline_spec.rb

@@ -954,7 +954,10 @@ describe Ci::Pipeline, :mailer do
 
       context 'when using legacy stages' do
         before do
-          stub_feature_flags(ci_pipeline_persisted_stages: false)
+          stub_feature_flags(
+            ci_pipeline_persisted_stages: false,
+            ci_atomic_processing: false
+          )
         end
 
         it 'returns legacy stages in valid order' do
@@ -962,9 +965,40 @@ describe Ci::Pipeline, :mailer do
         end
       end
 
+      context 'when using atomic processing' do
+        before do
+          stub_feature_flags(
+            ci_atomic_processing: true
+          )
+        end
+
+        context 'when pipelines is not complete' do
+          it 'returns stages in valid order' do
+            expect(subject).to all(be_a Ci::Stage)
+            expect(subject.map(&:name))
+              .to eq %w[sanity build test deploy cleanup]
+          end
+        end
+
+        context 'when pipeline is complete' do
+          before do
+            pipeline.succeed!
+          end
+
+          it 'returns stages in valid order' do
+            expect(subject).to all(be_a Ci::Stage)
+            expect(subject.map(&:name))
+              .to eq %w[sanity build test deploy cleanup]
+          end
+        end
+      end
+
       context 'when using persisted stages' do
         before do
-          stub_feature_flags(ci_pipeline_persisted_stages: true)
+          stub_feature_flags(
+            ci_pipeline_persisted_stages: true,
+            ci_atomic_processing: false
+          )
         end
 
         context 'when pipelines is not complete' do