Commit f35cf342 authored by Etienne Baqué's avatar Etienne Baqué

Merge branch 'support-ci-resource-group-in-cross-project-pipeline' into 'master'

Support Resrouce Group in Cross-Project/Parent-Child pipelines

See merge request gitlab-org/gitlab!53007
parents 41ea2bc0 7b83426d
......@@ -27,7 +27,7 @@ module Ci
# rubocop:enable Cop/ActiveRecordSerialize
state_machine :status do
after_transition [:created, :manual] => :pending do |bridge|
after_transition [:created, :manual, :waiting_for_resource] => :pending do |bridge|
next unless bridge.downstream_project
bridge.run_after_commit do
......@@ -156,6 +156,10 @@ module Ci
false
end
def any_unmet_prerequisites?
false
end
def expanded_environment_name
end
......
---
title: Pipeline-level concurrency control with Cross-Project/Parent-Child pipelines
merge_request: 53007
author:
type: added
......@@ -3924,6 +3924,60 @@ It can't start or end with `/`.
For more information, see [Deployments Safety](../environments/deployment_safety.md).
#### Pipeline-level concurrency control with Cross-Project/Parent-Child pipelines
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/39057) in GitLab 13.9.
You can define `resource_group` for downstream pipelines that are sensitive to concurrent
executions. The [`trigger` keyword](#trigger) can trigger downstream pipelines. The
[`resource_group` keyword](#resource_group) can co-exist with it. This is useful to control the
concurrency for deployment pipelines, while running non-sensitive jobs concurrently.
This example has two pipeline configurations in a project. When a pipeline starts running,
non-sensitive jobs are executed first and aren't affected by concurrent executions in other
pipelines. However, GitLab ensures that there are no other deployment pipelines running before
triggering a deployment (child) pipeline. If other deployment pipelines are running, GitLab waits
until those pipelines finish before running another one.
```yaml
# .gitlab-ci.yml (parent pipeline)
build:
stage: build
script: echo "Building..."
test:
stage: test
script: echo "Testing..."
deploy:
stage: deploy
trigger:
include: deploy.gitlab-ci.yml
strategy: depend
resource_group: AWS-production
```
```yaml
# deploy.gitlab-ci.yml (child pipeline)
stages:
- provision
- deploy
provision:
stage: provision
script: echo "Provisioning..."
deployment:
stage: deploy
script: echo "Deploying..."
```
Note that you must define [`strategy: depend`](#linking-pipelines-with-triggerstrategy)
with the `trigger` keyword. This ensures that the lock isn't released until the downstream pipeline
finishes.
### `release`
> [Introduced](https://gitlab.com/gitlab-org/gitlab/merge_requests/19298) in GitLab 13.2.
......
......@@ -73,17 +73,28 @@ module Gitlab
def to_resource
strong_memoize(:resource) do
if bridge?
::Ci::Bridge.new(attributes)
else
::Ci::Build.new(attributes).tap do |build|
build.assign_attributes(self.class.environment_attributes_for(build))
build.resource_group = Seed::Build::ResourceGroup.new(build, @resource_group_key).to_resource
end
processable = initialize_processable
assign_resource_group(processable)
processable
end
end
def initialize_processable
if bridge?
::Ci::Bridge.new(attributes)
else
::Ci::Build.new(attributes).tap do |build|
build.assign_attributes(self.class.environment_attributes_for(build))
end
end
end
def assign_resource_group(processable)
processable.resource_group =
Seed::Processable::ResourceGroup.new(processable, @resource_group_key)
.to_resource
end
def self.environment_attributes_for(build)
return {} unless build.has_environment?
......
......@@ -4,7 +4,7 @@ module Gitlab
module Ci
module Pipeline
module Seed
class Build
module Processable
class ResourceGroup < Seed::Base
include Gitlab::Utils::StrongMemoize
......
......@@ -8,6 +8,7 @@ module Gitlab
def self.extended_statuses
[[Status::Bridge::Failed],
[Status::Bridge::Manual],
[Status::Bridge::WaitingForResource],
[Status::Bridge::Play],
[Status::Bridge::Action]]
end
......
# frozen_string_literal: true
module Gitlab
module Ci
module Status
module Bridge
class WaitingForResource < Status::Processable::WaitingForResource
end
end
end
end
end
......@@ -4,22 +4,7 @@ module Gitlab
module Ci
module Status
module Build
class WaitingForResource < Status::Extended
##
# TODO: image is shared with 'pending'
# until we get a dedicated one
#
def illustration
{
image: 'illustrations/pending_job_empty.svg',
size: 'svg-430',
title: _('This job is waiting for resource: ') + subject.resource_group.key
}
end
def self.matches?(build, _)
build.waiting_for_resource?
end
class WaitingForResource < Status::Processable::WaitingForResource
end
end
end
......
# frozen_string_literal: true
module Gitlab
module Ci
module Status
module Processable
class WaitingForResource < Status::Extended
##
# TODO: image is shared with 'pending'
# until we get a dedicated one
#
def illustration
{
image: 'illustrations/pending_job_empty.svg',
size: 'svg-430',
title: _('This job is waiting for resource: ') + subject.resource_group.key
}
end
def self.matches?(processable, _)
processable.waiting_for_resource?
end
end
end
end
end
end
# frozen_string_literal: true
FactoryBot.define do
factory :ci_bridge, class: 'Ci::Bridge' do
factory :ci_bridge, class: 'Ci::Bridge', parent: :ci_processable do
name { 'bridge' }
stage { 'test' }
stage_idx { 0 }
ref { 'master' }
tag { false }
created_at { '2013-10-29 09:50:00 CET' }
status { :created }
scheduling_type { 'stage' }
pipeline factory: :ci_pipeline
trait :variables do
yaml_variables do
......
......@@ -3,15 +3,10 @@
include ActionDispatch::TestProcess
FactoryBot.define do
factory :ci_build, class: 'Ci::Build' do
factory :ci_build, class: 'Ci::Build', parent: :ci_processable do
name { 'test' }
stage { 'test' }
stage_idx { 0 }
ref { 'master' }
tag { false }
add_attribute(:protected) { false }
created_at { 'Di 29. Okt 09:50:00 CET 2013' }
scheduling_type { 'stage' }
pending
options do
......@@ -28,7 +23,6 @@ FactoryBot.define do
]
end
pipeline factory: :ci_pipeline
project { pipeline.project }
trait :degenerated do
......@@ -79,10 +73,6 @@ FactoryBot.define do
status { 'created' }
end
trait :waiting_for_resource do
status { 'waiting_for_resource' }
end
trait :preparing do
status { 'preparing' }
end
......@@ -213,14 +203,6 @@ FactoryBot.define do
trigger_request factory: :ci_trigger_request
end
trait :resource_group do
waiting_for_resource_at { 5.minutes.ago }
after(:build) do |build, evaluator|
build.resource_group = create(:ci_resource_group, project: build.project)
end
end
trait :with_deployment do
after(:build) do |build, evaluator|
##
......
# frozen_string_literal: true
FactoryBot.define do
factory :ci_processable, class: 'Ci::Processable' do
name { 'processable' }
stage { 'test' }
stage_idx { 0 }
ref { 'master' }
tag { false }
pipeline factory: :ci_pipeline
project { pipeline.project }
scheduling_type { 'stage' }
trait :waiting_for_resource do
status { 'waiting_for_resource' }
end
trait :resource_group do
waiting_for_resource_at { 5.minutes.ago }
after(:build) do |processable, evaluator|
processable.resource_group = create(:ci_resource_group, project: processable.project)
end
end
end
end
......@@ -846,6 +846,28 @@ RSpec.describe 'Pipeline', :js do
end
end
end
context 'when deploy job is a bridge to trigger a downstream pipeline' do
let!(:deploy_job) do
create(:ci_bridge, :created, stage: 'deploy', name: 'deploy',
stage_idx: 2, pipeline: pipeline, project: project, resource_group: resource_group)
end
it 'shows deploy job as waiting for resource' do
subject
within('.pipeline-header-container') do
expect(page).to have_content('waiting')
end
within('.pipeline-graph') do
within '.stage-column:nth-child(2)' do
expect(page).to have_content('deploy')
expect(page).to have_css('.ci-status-icon-waiting-for-resource')
end
end
end
end
end
end
end
......
......@@ -383,14 +383,25 @@ RSpec.describe Gitlab::Ci::Pipeline::Seed::Build do
end
context 'when job is a bridge' do
let(:attributes) do
let(:base_attributes) do
{
name: 'rspec', ref: 'master', options: { trigger: 'my/project' }, scheduling_type: :stage
}
end
let(:attributes) { base_attributes }
it { is_expected.to be_a(::Ci::Bridge) }
it { is_expected.to be_valid }
context 'when job belongs to a resource group' do
let(:attributes) { base_attributes.merge(resource_group_key: 'iOS') }
it 'returns a job with resource group' do
expect(subject.resource_group).not_to be_nil
expect(subject.resource_group.key).to eq('iOS')
end
end
end
it 'memoizes a resource object' do
......
......@@ -2,7 +2,7 @@
require 'spec_helper'
RSpec.describe Gitlab::Ci::Pipeline::Seed::Build::ResourceGroup do
RSpec.describe Gitlab::Ci::Pipeline::Seed::Processable::ResourceGroup do
let_it_be(:project) { create(:project) }
let(:job) { build(:ci_build, project: project) }
let(:seed) { described_class.new(job, resource_group_key) }
......
......@@ -117,14 +117,31 @@ RSpec.describe Gitlab::Ci::Status::Bridge::Factory do
end
end
context 'when bridge is waiting for resource' do
let(:bridge) { create_bridge(:waiting_for_resource, :resource_group) }
it 'matches correct core status' do
expect(factory.core_status).to be_a Gitlab::Ci::Status::WaitingForResource
end
it 'fabricates status with correct details' do
expect(status.text).to eq 'waiting'
expect(status.group).to eq 'waiting-for-resource'
expect(status.icon).to eq 'status_pending'
expect(status.favicon).to eq 'favicon_pending'
expect(status.illustration).to include(:image, :size, :title)
expect(status).not_to have_details
end
end
private
def create_bridge(trait)
def create_bridge(*traits)
upstream_project = create(:project, :repository)
downstream_project = create(:project, :repository)
upstream_pipeline = create(:ci_pipeline, :running, project: upstream_project)
trigger = { trigger: { project: downstream_project.full_path, branch: 'feature' } }
create(:ci_bridge, trait, options: trigger, pipeline: upstream_pipeline)
create(:ci_bridge, *traits, options: trigger, pipeline: upstream_pipeline)
end
end
# frozen_string_literal: true
require 'fast_spec_helper'
RSpec.describe Gitlab::Ci::Status::Bridge::WaitingForResource do
it { expect(described_class).to be < Gitlab::Ci::Status::Processable::WaitingForResource }
end
# frozen_string_literal: true
require 'fast_spec_helper'
RSpec.describe Gitlab::Ci::Status::Build::WaitingForResource do
it { expect(described_class).to be < Gitlab::Ci::Status::Processable::WaitingForResource }
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Gitlab::Ci::Status::Processable::WaitingForResource do
let(:user) { create(:user) }
subject do
processable = create(:ci_build, :waiting_for_resource, :resource_group)
described_class.new(Gitlab::Ci::Status::Core.new(processable, user))
end
describe '#illustration' do
it { expect(subject.illustration).to include(:image, :size, :title) }
end
describe '.matches?' do
subject {described_class.matches?(processable, user) }
context 'when processable is waiting for resource' do
let(:processable) { create(:ci_build, :waiting_for_resource) }
it 'is a correct match' do
expect(subject).to be true
end
end
context 'when processable is not waiting for resource' do
let(:processable) { create(:ci_build) }
it 'does not match' do
expect(subject).to be false
end
end
end
end
......@@ -80,6 +80,14 @@ RSpec.describe Ci::Bridge do
end
end
it "schedules downstream pipeline creation when the status is waiting for resource" do
bridge.status = :waiting_for_resource
expect(bridge).to receive(:schedule_downstream_pipeline!)
bridge.enqueue_waiting_for_resource!
end
it 'raises error when the status is failed' do
bridge.status = :failed
......
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::CreatePipelineService, '#execute' do
let_it_be(:group) { create(:group, name: 'my-organization') }
let(:upstream_project) { create(:project, :repository, name: 'upstream', group: group) }
let(:downstram_project) { create(:project, :repository, name: 'downstream', group: group) }
let(:user) { create(:user) }
let(:service) do
described_class.new(upstream_project, user, ref: 'master')
end
before do
upstream_project.add_developer(user)
downstram_project.add_developer(user)
create_gitlab_ci_yml(upstream_project, upstream_config)
create_gitlab_ci_yml(downstram_project, downstream_config)
end
context 'with resource group', :aggregate_failures do
let(:upstream_config) do
<<~YAML
instrumentation_test:
stage: test
resource_group: iOS
trigger:
project: my-organization/downstream
strategy: depend
YAML
end
let(:downstream_config) do
<<~YAML
test:
script: echo "Testing..."
YAML
end
it 'creates bridge job with resource group' do
pipeline = create_pipeline!
test = pipeline.statuses.find_by(name: 'instrumentation_test')
expect(pipeline).to be_created_successfully
expect(pipeline.triggered_pipelines).not_to be_exist
expect(upstream_project.resource_groups.count).to eq(1)
expect(test).to be_a Ci::Bridge
expect(test).to be_waiting_for_resource
expect(test.resource_group.key).to eq('iOS')
end
context 'when sidekiq processes the job', :sidekiq_inline do
it 'transitions to pending status and triggers a downstream pipeline' do
pipeline = create_pipeline!
test = pipeline.statuses.find_by(name: 'instrumentation_test')
expect(test).to be_pending
expect(pipeline.triggered_pipelines.count).to eq(1)
end
context 'when the resource is occupied by the other bridge' do
before do
resource_group = create(:ci_resource_group, project: upstream_project, key: 'iOS')
resource_group.assign_resource_to(create(:ci_build, project: upstream_project))
end
it 'stays waiting for resource' do
pipeline = create_pipeline!
test = pipeline.statuses.find_by(name: 'instrumentation_test')
expect(test).to be_waiting_for_resource
expect(pipeline.triggered_pipelines.count).to eq(0)
end
end
end
end
def create_pipeline!
service.execute(:push)
end
def create_gitlab_ci_yml(project, content)
project.repository.create_file(user, '.gitlab-ci.yml', content, branch_name: 'master', message: 'test')
end
end
......@@ -84,21 +84,46 @@ RSpec.describe Ci::CreatePipelineService, '#execute' do
stage: test
resource_group: iOS
trigger:
include:
- local: path/to/child.yml
include: path/to/child.yml
strategy: depend
YAML
end
# TODO: This test will be properly implemented in the next MR
# for https://gitlab.com/gitlab-org/gitlab/-/issues/39057.
it 'creates bridge job but still resource group is no-op', :aggregate_failures do
it 'creates bridge job with resource group', :aggregate_failures do
pipeline = create_pipeline!
test = pipeline.statuses.find_by(name: 'instrumentation_test')
expect(pipeline).to be_persisted
expect(pipeline).to be_created_successfully
expect(pipeline.triggered_pipelines).not_to be_exist
expect(project.resource_groups.count).to eq(1)
expect(test).to be_a Ci::Bridge
expect(project.resource_groups.count).to eq(0)
expect(test).to be_waiting_for_resource
expect(test.resource_group.key).to eq('iOS')
end
context 'when sidekiq processes the job', :sidekiq_inline do
it 'transitions to pending status and triggers a downstream pipeline' do
pipeline = create_pipeline!
test = pipeline.statuses.find_by(name: 'instrumentation_test')
expect(test).to be_pending
expect(pipeline.triggered_pipelines.count).to eq(1)
end
context 'when the resource is occupied by the other bridge' do
before do
resource_group = create(:ci_resource_group, project: project, key: 'iOS')
resource_group.assign_resource_to(create(:ci_build, project: project))
end
it 'stays waiting for resource' do
pipeline = create_pipeline!
test = pipeline.statuses.find_by(name: 'instrumentation_test')
expect(test).to be_waiting_for_resource
expect(pipeline.triggered_pipelines.count).to eq(0)
end
end
end
end
end
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment