Support FIFO/LIFO process modes to Resource Group

This commit adds the support to process jobs in
FIFO/LIFO order, aside from the semaphore mode
(i.e. non deterministic order).

Changelog: added

app/models/ci/resource_group.rb

@@ -14,6 +14,12 @@ module Ci
 
     before_create :ensure_resource
 
+    enum process_mode: {
+      unordered: 0,
+      oldest_first: 1,
+      newest_first: 2
+    }
+
     ##
     # NOTE: This is concurrency-safe method that the subquery in the `UPDATE`
     # works as explicit locking.
@@ -25,8 +31,34 @@ module Ci
       resources.retained_by(processable).update_all(build_id: nil) > 0
     end
 
+    def upcoming_processables
+      if unordered? || Feature.disabled?(:ci_resource_group_process_modes, project, default_enabled: :yaml)
+        processables.waiting_for_resource
+      elsif oldest_first?
+        processables.waiting_for_resource_or_upcoming
+          .order(Arel.sql("commit_id ASC, #{sort_by_job_status}"))
+      elsif newest_first?
+        processables.waiting_for_resource_or_upcoming
+          .order(Arel.sql("commit_id DESC, #{sort_by_job_status}"))
+      else
+        Ci::Processable.none
+      end
+    end
+
     private
 
+    # In order to avoid deadlock, we do NOT specify the job execution order in the same pipeline.
+    # The system processes wherever ready to transition to `pending` status from `waiting_for_resource`.
+    # See https://gitlab.com/gitlab-org/gitlab/-/issues/202186 for more information.
+    def sort_by_job_status
+      <<~SQL
+        CASE status
+          WHEN 'waiting_for_resource' THEN 0
+          ELSE 1
+        END ASC
+      SQL
+    end
+
     def ensure_resource
       # Currently we only support one resource per group, which means
       # maximum one build can be set to the resource group, thus builds

app/models/concerns/ci/has_status.rb

@@ -95,6 +95,7 @@ module Ci
       scope :failed_or_canceled, -> { with_status(:failed, :canceled) }
       scope :complete, -> { with_status(completed_statuses) }
       scope :incomplete, -> { without_statuses(completed_statuses) }
+      scope :waiting_for_resource_or_upcoming, -> { with_status(:created, :scheduled, :waiting_for_resource) }
 
       scope :cancelable, -> do
         where(status: [:running, :waiting_for_resource, :preparing, :pending, :created, :scheduled])

app/policies/ci/resource_group_policy.rb

+# frozen_string_literal: true
+
+module Ci
+  class ResourceGroupPolicy < BasePolicy
+    delegate { @subject.project }
+  end
+end

app/policies/project_policy.rb

@@ -357,6 +357,8 @@ class ProjectPolicy < BasePolicy
     enable :update_commit_status
     enable :create_build
     enable :update_build
+    enable :read_resource_group
+    enable :update_resource_group
     enable :create_merge_request_from
     enable :create_wiki
     enable :push_code

app/services/ci/resource_groups/assign_resource_from_resource_group_service.rb

@@ -9,7 +9,7 @@ module Ci
 
         free_resources = resource_group.resources.free.count
 
-        resource_group.processables.waiting_for_resource.take(free_resources).each do |processable|
+        resource_group.upcoming_processables.take(free_resources).each do |processable|
           processable.enqueue_waiting_for_resource
         end
       end

config/feature_flags/development/ci_resource_group_process_modes.yml

+---
+name: ci_resource_group_process_modes
+introduced_by_url: https://gitlab.com/gitlab-org/gitlab/-/merge_requests/67015
+rollout_issue_url: https://gitlab.com/gitlab-org/gitlab/-/issues/340380
+milestone: '14.3'
+type: development
+group: group::release
+default_enabled: false

db/migrate/20210916132547_add_process_mode_to_resource_groups.rb

+# frozen_string_literal: true
+
+class AddProcessModeToResourceGroups < Gitlab::Database::Migration[1.0]
+  enable_lock_retries!
+
+  PROCESS_MODE_UNORDERED = 0
+
+  def up
+    add_column :ci_resource_groups, :process_mode, :integer, default: PROCESS_MODE_UNORDERED, null: false, limit: 2
+  end
+
+  def down
+    remove_column :ci_resource_groups, :process_mode
+  end
+end

db/schema_migrations/20210916132547

+d0953fdbaa6cf656e298ea482b3e3f931254276cb2285cffafba3d94b0626d3f
\ No newline at end of file

db/structure.sql

@@ -11914,7 +11914,8 @@ CREATE TABLE ci_resource_groups (
     created_at timestamp with time zone NOT NULL,
     updated_at timestamp with time zone NOT NULL,
     project_id bigint NOT NULL,
-    key character varying(255) NOT NULL
+    key character varying(255) NOT NULL,
+    process_mode smallint DEFAULT 0 NOT NULL
 );
 
 CREATE SEQUENCE ci_resource_groups_id_seq

doc/api/resource_groups.md

+---
+stage: Release
+group: Release
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
+type: concepts, howto
+---
+
+# Resource Groups API
+
+You can read more about [controling the job concurrency with resource groups](../ci/resource_groups/index.md).
+
+## Get a specific resource group
+
+```plaintext
+GET /projects/:id/resource_groups/:key
+```
+
+| Attribute | Type    | Required | Description         |
+|-----------|---------|----------|---------------------|
+| `id`      | integer/string     | yes      | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user |
+| `key`     | string  | yes      | The key of the resource group |
+
+```shell
+curl --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/resource_groups/production"
+```
+
+Example of response
+
+```json
+{
+  "id": 3,
+  "key": "production",
+  "process_mode": "unordered",
+  "created_at": "2021-09-01T08:04:59.650Z",
+  "updated_at": "2021-09-01T08:04:59.650Z"
+}
+```
+
+## Edit an existing resource group
+
+Updates an existing resource group's properties.
+
+It returns `200` if the resource group was successfully updated. In case of an error, a status code `400` is returned.
+
+```plaintext
+PUT /projects/:id/resource_groups/:key
+```
+
+| Attribute       | Type    | Required                          | Description                      |
+| --------------- | ------- | --------------------------------- | -------------------------------  |
+| `id`            | integer/string | yes                        | The ID or [URL-encoded path of the project](index.md#namespaced-path-encoding) owned by the authenticated user            |
+| `key`           | string  | yes                               | The key of the resource group |
+| `process_mode`  | string  | no                                | The process mode of the resource group. One of `unordered`, `oldest_first` or `newest_first`. Read [process modes](../ci/resource_groups/index.md#process-modes) for more information. |
+
+```shell
+curl --request PUT --data "process_mode=oldest_first" \
+     --header "PRIVATE-TOKEN: <your_access_token>" "https://gitlab.example.com/api/v4/projects/1/resource_groups/production"
+```
+
+Example response:
+
+```json
+{
+  "id": 3,
+  "key": "production",
+  "process_mode": "oldest_first",
+  "created_at": "2021-09-01T08:04:59.650Z",
+  "updated_at": "2021-09-01T08:13:38.679Z"
+}
+```

doc/ci/environments/deployment_safety.md

@@ -60,7 +60,7 @@ The improved pipeline flow **after** using the resource group:
 1. `deploy` job in Pipeline-A finishes.
 1. `deploy` job in Pipeline-B starts running.
 
-For more information, see [`resource_group` keyword in `.gitlab-ci.yml`](../yaml/index.md#resource_group).
+For more information, see [Resource Group documentation](../resource_groups/index.md).
 
 ## Skip outdated deployment jobs
 

doc/ci/resource_groups/index.md

+---
+stage: Release
+group: Release
+info: To determine the technical writer assigned to the Stage/Group associated with this page, see https://about.gitlab.com/handbook/engineering/ux/technical-writing/#assignments
+description: Control the job concurrency in GitLab CI/CD
+---
+
+# Resource Group **(FREE)**
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/15536) in GitLab 12.7.
+
+By default, pipelines in GitLab CI/CD run in parallel. The parallelization is an important factor to improve
+the feedback loop in merge requests, however, there are some situations that
+you may want to limit the concurrency on deployment
+jobs to run them one by one.
+Resource Group allows you to strategically control
+the concurrency of the jobs for optimizing your continuous deployments workflow with safety.
+
+## Add a resource group
+
+Provided that you have the following pipeline configuration (`.gitlab-ci.yml` file in your repository):
+
+```yaml
+build:
+  stage: build
+  script: echo "Your build script"
+
+deploy:
+  stage: deploy
+  script: echo "Your deployment script"
+  environment: production
+```
+
+Every time you push a new commit to a branch, it runs a new pipeline that has
+two jobs `build` and `deploy`. But if you push multiple commits in a short interval, multiple
+pipelines start running simultaneously, for example:
+
+- The first pipeline runs the jobs `build` -> `deploy`
+- The second pipeline runs the jobs `build` -> `deploy`
+
+In this case, the `deploy` jobs across different pipelines could run concurrently
+to the `production` environment. Running multiple deployment scripts to the same
+infrastructure could harm/confuse the instance and leave it in a corrupted state in the worst case.
+
+In order to ensure that a `deploy` job runs once at a time, you can specify
+[`resource_group` keyword](../yaml/index.md#resource_group) to the concurrency sensitive job:
+
+```yaml
+deploy:
+  ...
+  resource_group: production
+```
+
+With this configuration, the safety on the deployments is assured while you
+can still run `build` jobs concurrently for maximizing the pipeline efficency.
+
+## Requirements
+
+- The basic knowledge of the [GitLab CI/CD pipelines](../pipelines/index.md)
+- The basic knowledge of the [GitLab Environments and Deployments](../environments/index.md)
+- [Developer role](../../user/permissions.md) (or above) in the project to configure CI/CD pipelines.
+
+### Limitations
+
+Only one resource can be attached to a resource group.
+
+## Process modes
+
+> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/202186) in GitLab 14.3.
+
+FLAG:
+On self-managed GitLab, by default this feature is not available.
+To make it available, ask an administrator to [enable the `ci_resource_group_process_modes` flag](../../administration/feature_flags.md).
+On GitLab.com, this feature is not available.
+The feature is not ready for production use.
+
+You can choose a process mode to strategically control the job concurrency for your deployment preferences.
+The following modes are supported:
+
+- **Unordered:** This is the default process mode that limits the concurrency on running jobs.
+  It's the easiest option to use and useful when you don't care about the execution order
+  of the jobs. It starts processing the jobs whenever a job ready to run.
+- **Oldest first:** This process mode limits the concurrency of the jobs. When a resource is free,
+  it picks the first job from the list of upcoming jobs (`created`, `scheduled`, or `waiting_for_resource` state)
+  that are sorted by pipeline ID in ascending order.
+
+  This mode is useful when you want to ensure that the jobs are executed from the oldest pipeline.
+  This is less efficient compared to the `unordered` mode in terms of the pipeline efficiency,
+  but safer for continuous deployments.
+
+- **Newest first:** This process mode limits the concurrency of the jobs. When a resource is free,
+  it picks the first job from the list of upcoming jobs (`created`, `scheduled` or `waiting_for_resource` state)
+  that are sorted by pipeline ID in descending order.
+
+  This mode is useful when you want to ensure that the jobs are executed from the newest pipeline and
+  cancel all of the old deploy jobs with the [skip outdated deployment jobs](../environments/deployment_safety.md#skip-outdated-deployment-jobs) feature.
+  This is the most efficient option in terms of the pipeline efficiency, but you must ensure that each deployment job is idempotent.
+
+### Change the process mode
+
+To change the process mode of a resource group, you need to use the API and
+send a request to [edit an existing resource group](../../api/resource_groups.md#edit-an-existing-resource-group)
+by specifying the `process_mode`:
+
+- `unordered`
+- `oldest_first`
+- `newest_first`
+
+### An example of difference between the process modes
+
+Consider the following `.gitlab-ci.yml`, where we have two jobs `build` and `deploy`
+each running in their own stage, and the `deploy` job has a resource group set to
+`production`:
+
+```yaml
+build:
+  stage: build
+  script: echo "Your build script"
+
+deploy:
+  stage: deploy
+  script: echo "Your deployment script"
+  environment: production
+  resource_group: production
+```
+
+If three commits are pushed to the project in a short interval, that means that three
+pipelines run almost at the same time:
+
+- The first pipeline runs the jobs `build` -> `deploy`. Let's call this deployment job `deploy-1`.
+- The second pipeline runs the jobs `build` -> `deploy`. Let's call this deployment job `deploy-2`.
+- The third pipeline runs the jobs `build` -> `deploy`. Let's call this deployment job `deploy-3`.
+
+Depending on the process mode of the resource group:
+
+- If the process mode is set to `unordered`:
+  - `deploy-1`, `deploy-2`, and `deploy-3` do not run in parallel.
+  - There is no guarantee on the job execution order, for example, `deploy-1` could run before or after `deploy-3` runs.
+- If the process mode is `oldest_first`:
+  - `deploy-1`, `deploy-2`, and `deploy-3` do not run in parallel.
+  - `deploy-1` runs first, `deploy-2` runs second, and `deploy-3` runs last.
+- If the process mode is `newest_first`:
+  - `deploy-1`, `deploy-2`, and `deploy-3` do not run in parallel.
+  - `deploy-3` runs first, `deploy-2` runs second and `deploy-1` runs last.
+
+## Pipeline-level concurrency control with Cross-Project/Parent-Child pipelines
+
+See the how to [control the pipeline concurrency in cross-project pipelines](../yaml/index.md#pipeline-level-concurrency-control-with-cross-projectparent-child-pipelines).
+
+## API
+
+See the [API documentation](../../api/resource_groups.md).
+
+## Related features
+
+Read more how you can use GitLab for [safe deployments](../environments/deployment_safety.md).
+
+<!-- ## Troubleshooting
+
+Include any troubleshooting steps that you can foresee. If you know beforehand what issues
+one might have when setting this up, or when something is changed, or on upgrading, it's
+important to describe those, too. Think of things that may go wrong and include them here.
+This is important to minimize requests for support, and to avoid doc comments with
+questions that you know someone might ask.
+
+Each scenario can be a third-level heading, e.g. `### Getting error message X`.
+If you have none to add when creating a doc, leave this section in place
+but commented out to help encourage others to add to it in the future. -->

doc/ci/yaml/index.md

@@ -3859,7 +3859,7 @@ can be deployed to, but there can be only one deployment per device at any given
 The `resource_group` value can only contain letters, digits, `-`, `_`, `/`, `$`, `{`, `}`, `.`, and spaces.
 It can't start or end with `/`.
 
-For more information, see [Deployments Safety](../environments/deployment_safety.md).
+For more information, see [Resource Group documentation](../resource_groups/index.md).
 
 #### Pipeline-level concurrency control with Cross-Project/Parent-Child pipelines
 

lib/api/api.rb

@@ -157,6 +157,7 @@ module API
       mount ::API::Ci::Jobs
       mount ::API::Ci::Pipelines
       mount ::API::Ci::PipelineSchedules
+      mount ::API::Ci::ResourceGroups
       mount ::API::Ci::Runner
       mount ::API::Ci::Runners
       mount ::API::Ci::Triggers

lib/api/ci/resource_groups.rb

+# frozen_string_literal: true
+
+module API
+  module Ci
+    class ResourceGroups < ::API::Base
+      before { authenticate! }
+
+      feature_category :continuous_delivery
+
+      params do
+        requires :id, type: String, desc: 'The ID of a project'
+      end
+      resource :projects, requirements: ::API::API::NAMESPACE_OR_PROJECT_REQUIREMENTS do
+        desc 'Get a single resource group' do
+          success Entities::Ci::ResourceGroup
+        end
+        params do
+          requires :key, type: String, desc: 'The key of the resource group'
+        end
+        get ':id/resource_groups/:key' do
+          authorize! :read_resource_group, resource_group
+
+          present resource_group, with: Entities::Ci::ResourceGroup
+        end
+
+        desc 'Edit a resource group' do
+          success Entities::Ci::ResourceGroup
+        end
+        params do
+          requires :key, type: String, desc: 'The key of the resource group'
+          optional :process_mode, type: String, desc: 'The process mode',
+                                  values: ::Ci::ResourceGroup.process_modes.keys
+        end
+        put ':id/resource_groups/:key' do
+          not_found! unless ::Feature.enabled?(:ci_resource_group_process_modes, user_project, default_enabled: :yaml)
+          authorize! :update_resource_group, resource_group
+
+          if resource_group.update(declared_params(include_missing: false))
+            present resource_group, with: Entities::Ci::ResourceGroup
+          else
+            render_validation_error!(resource_group)
+          end
+        end
+      end
+
+      helpers do
+        def resource_group
+          @resource_group ||= user_project.resource_groups.find_by_key!(params[:key])
+        end
+      end
+    end
+  end
+end

lib/api/entities/ci/resource_group.rb

+# frozen_string_literal: true
+
+module API
+  module Entities
+    module Ci
+      class ResourceGroup < Grape::Entity
+        expose :id, :key, :process_mode, :created_at, :updated_at
+      end
+    end
+  end
+end

spec/models/ci/resource_group_spec.rb

@@ -85,4 +85,77 @@ RSpec.describe Ci::ResourceGroup do
       end
     end
   end
+
+  describe '#upcoming_processables' do
+    subject { resource_group.upcoming_processables }
+
+    let_it_be(:project) { create(:project, :repository) }
+    let_it_be(:pipeline_1) { create(:ci_pipeline, project: project) }
+    let_it_be(:pipeline_2) { create(:ci_pipeline, project: project) }
+
+    let!(:resource_group) { create(:ci_resource_group, process_mode: process_mode, project: project) }
+
+    Ci::HasStatus::STATUSES_ENUM.keys.each do |status|
+      let!("build_1_#{status}") { create(:ci_build, pipeline: pipeline_1, status: status, resource_group: resource_group) }
+      let!("build_2_#{status}") { create(:ci_build, pipeline: pipeline_2, status: status, resource_group: resource_group) }
+    end
+
+    context 'when process mode is unordered' do
+      let(:process_mode) { :unordered }
+
+      it 'returns correct jobs in an indeterministic order' do
+        expect(subject).to contain_exactly(build_1_waiting_for_resource, build_2_waiting_for_resource)
+      end
+    end
+
+    context 'when process mode is oldest_first' do
+      let(:process_mode) { :oldest_first }
+
+      it 'returns correct jobs in a specific order' do
+        expect(subject[0]).to eq(build_1_waiting_for_resource)
+        expect(subject[1..2]).to contain_exactly(build_1_created, build_1_scheduled)
+        expect(subject[3]).to eq(build_2_waiting_for_resource)
+        expect(subject[4..5]).to contain_exactly(build_2_created, build_2_scheduled)
+      end
+
+      context 'when ci_resource_group_process_modes feature flag is disabled' do
+        it 'returns correct jobs in an indeterministic order' do
+          stub_feature_flags(ci_resource_group_process_modes: false)
+
+          expect(subject).to contain_exactly(build_1_waiting_for_resource, build_2_waiting_for_resource)
+        end
+      end
+    end
+
+    context 'when process mode is newest_first' do
+      let(:process_mode) { :newest_first }
+
+      it 'returns correct jobs in a specific order' do
+        expect(subject[0]).to eq(build_2_waiting_for_resource)
+        expect(subject[1..2]).to contain_exactly(build_2_created, build_2_scheduled)
+        expect(subject[3]).to eq(build_1_waiting_for_resource)
+        expect(subject[4..5]).to contain_exactly(build_1_created, build_1_scheduled)
+      end
+
+      context 'when ci_resource_group_process_modes feature flag is disabled' do
+        it 'returns correct jobs in an indeterministic order' do
+          stub_feature_flags(ci_resource_group_process_modes: false)
+
+          expect(subject).to contain_exactly(build_1_waiting_for_resource, build_2_waiting_for_resource)
+        end
+      end
+    end
+
+    context 'when process mode is unknown' do
+      let(:process_mode) { :unordered }
+
+      before do
+        resource_group.update_column(:process_mode, 3)
+      end
+
+      it 'returns empty' do
+        is_expected.to be_empty
+      end
+    end
+  end
 end

spec/models/concerns/ci/has_status_spec.rb

@@ -363,6 +363,18 @@ RSpec.describe Ci::HasStatus do
         it_behaves_like 'not containing the job', status
       end
     end
+
+    describe '.waiting_for_resource_or_upcoming' do
+      subject { CommitStatus.waiting_for_resource_or_upcoming }
+
+      %i[created scheduled waiting_for_resource].each do |status|
+        it_behaves_like 'containing the job', status
+      end
+
+      %i[running failed success canceled].each do |status|
+        it_behaves_like 'not containing the job', status
+      end
+    end
   end
 
   describe '::DEFAULT_STATUS' do

spec/requests/api/ci/resource_groups_spec.rb

+# frozen_string_literal: true
+
+require 'spec_helper'
+
+RSpec.describe API::Ci::ResourceGroups do
+  let_it_be(:project) { create(:project) }
+  let_it_be(:developer) { create(:user).tap { |u| project.add_developer(u) } }
+  let_it_be(:reporter) { create(:user).tap { |u| project.add_reporter(u) } }
+
+  let(:user) { developer }
+
+  describe 'GET /projects/:id/resource_groups/:key' do
+    subject { get api("/projects/#{project.id}/resource_groups/#{key}", user) }
+
+    let!(:resource_group) { create(:ci_resource_group, project: project) }
+    let(:key) { resource_group.key }
+
+    it 'returns a resource group', :aggregate_failures do
+      subject
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response['id']).to eq(resource_group.id)
+      expect(json_response['key']).to eq(resource_group.key)
+      expect(json_response['process_mode']).to eq(resource_group.process_mode)
+      expect(Time.parse(json_response['created_at'])).to be_like_time(resource_group.created_at)
+      expect(Time.parse(json_response['updated_at'])).to be_like_time(resource_group.updated_at)
+    end
+
+    context 'when user is reporter' do
+      let(:user) { reporter }
+
+      it 'returns forbidden' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+
+    context 'when there is no corresponding resource group' do
+      let(:key) { 'unknown' }
+
+      it 'returns not found' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+  end
+
+  describe 'PUT /projects/:id/resource_groups/:key' do
+    subject { put api("/projects/#{project.id}/resource_groups/#{key}", user), params: params }
+
+    let!(:resource_group) { create(:ci_resource_group, project: project) }
+    let(:key) { resource_group.key }
+    let(:params) { { process_mode: :oldest_first } }
+
+    it 'changes the process mode of a resource group' do
+      expect { subject }
+        .to change { resource_group.reload.process_mode }.from('unordered').to('oldest_first')
+
+      expect(response).to have_gitlab_http_status(:ok)
+      expect(json_response['process_mode']).to eq('oldest_first')
+    end
+
+    context 'when ci_resource_group_process_modes feature flag is disabled' do
+      before do
+        stub_feature_flags(ci_resource_group_process_modes: false)
+      end
+
+      it 'returns not found' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+
+    context 'with invalid parameter' do
+      let(:params) { { process_mode: :unknown } }
+
+      it 'returns bad request' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:bad_request)
+      end
+    end
+
+    context 'when user is reporter' do
+      let(:user) { reporter }
+
+      it 'returns forbidden' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:forbidden)
+      end
+    end
+
+    context 'when there is no corresponding resource group' do
+      let(:key) { 'unknown' }
+
+      it 'returns not found' do
+        subject
+
+        expect(response).to have_gitlab_http_status(:not_found)
+      end
+    end
+  end
+end

spec/services/ci/resource_groups/assign_resource_from_resource_group_service_spec.rb

@@ -48,6 +48,92 @@ RSpec.describe Ci::ResourceGroups::AssignResourceFromResourceGroupService do
           expect(build).to be_pending
         end
       end
+
+      context 'when process mode is oldest_first' do
+        let(:resource_group) { create(:ci_resource_group, process_mode: :oldest_first, project: project) }
+
+        it 'requests resource' do
+          subject
+
+          expect(build.reload).to be_pending
+          expect(build.resource).to be_present
+        end
+
+        context 'when the other job exists in the newer pipeline' do
+          let!(:build_2) { create(:ci_build, :waiting_for_resource, project: project, user: user, resource_group: resource_group) }
+
+          it 'requests resource for the job in the oldest pipeline' do
+            subject
+
+            expect(build.reload).to be_pending
+            expect(build.resource).to be_present
+            expect(build_2.reload).to be_waiting_for_resource
+            expect(build_2.resource).to be_nil
+          end
+        end
+
+        context 'when build is not `waiting_for_resource` state' do
+          let!(:build) { create(:ci_build, :created, project: project, user: user, resource_group: resource_group) }
+
+          it 'attempts to request a resource' do
+            expect_next_found_instance_of(Ci::Build) do |job|
+              expect(job).to receive(:enqueue_waiting_for_resource).and_call_original
+            end
+
+            subject
+          end
+
+          it 'does not change the job status' do
+            subject
+
+            expect(build.reload).to be_created
+            expect(build.resource).to be_nil
+          end
+        end
+      end
+
+      context 'when process mode is newest_first' do
+        let(:resource_group) { create(:ci_resource_group, process_mode: :newest_first, project: project) }
+
+        it 'requests resource' do
+          subject
+
+          expect(build.reload).to be_pending
+          expect(build.resource).to be_present
+        end
+
+        context 'when the other job exists in the newer pipeline' do
+          let!(:build_2) { create(:ci_build, :waiting_for_resource, project: project, user: user, resource_group: resource_group) }
+
+          it 'requests resource for the job in the newest pipeline' do
+            subject
+
+            expect(build.reload).to be_waiting_for_resource
+            expect(build.resource).to be_nil
+            expect(build_2.reload).to be_pending
+            expect(build_2.resource).to be_present
+          end
+        end
+
+        context 'when build is not `waiting_for_resource` state' do
+          let!(:build) { create(:ci_build, :created, project: project, user: user, resource_group: resource_group) }
+
+          it 'attempts to request a resource' do
+            expect_next_found_instance_of(Ci::Build) do |job|
+              expect(job).to receive(:enqueue_waiting_for_resource).and_call_original
+            end
+
+            subject
+          end
+
+          it 'does not change the job status' do
+            subject
+
+            expect(build.reload).to be_created
+            expect(build.resource).to be_nil
+          end
+        end
+      end
     end
 
     context 'when there are no available resources' do

spec/support/shared_contexts/policies/project_policy_shared_context.rb

@@ -49,6 +49,7 @@ RSpec.shared_context 'ProjectPolicy context' do
       resolve_note update_build update_commit_status update_container_image
       update_deployment update_environment update_merge_request
       update_metrics_dashboard_annotation update_pipeline update_release destroy_release
+      read_resource_group update_resource_group
     ]
   end