Commit 1da0bf61 authored by Furkan Ayhan's avatar Furkan Ayhan Committed by Grzegorz Bizon

Add trigger support for matrix jobs

With this, users can define trigger jobs with parallel-matrix config.
parent 1d6eea46
---
title: Add trigger support for matrix jobs
merge_request: 55348
author:
type: added
......@@ -3664,6 +3664,40 @@ deploystacks:
- PROVIDER: [aws, ovh, gcp, vultr]
```
##### Parallel `matrix` trigger jobs
> [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/270957) in GitLab 13.10.
Use `matrix:` to run a [trigger](#trigger) job multiple times in parallel in a single pipeline,
but with different variable values for each instance of the job.
```yaml
deploystacks:
stage: deploy
trigger:
include: path/to/child-pipeline.yml
parallel:
matrix:
- PROVIDER: aws
STACK: [monitoring, app1]
- PROVIDER: ovh
STACK: [monitoring, backup]
- PROVIDER: [gcp, vultr]
STACK: [data]
```
This example generates 6 parallel `deploystacks` trigger jobs, each with different values
for `PROVIDER` and `STACK`, and they create 6 different child pipelines with those variables.
```plaintext
deploystacks: [aws, monitoring]
deploystacks: [aws, app1]
deploystacks: [ovh, monitoring]
deploystacks: [ovh, backup]
deploystacks: [gcp, data]
deploystacks: [vultr, data]
```
### `trigger`
> - [Introduced](https://gitlab.com/gitlab-org/gitlab/-/issues/8997) in [GitLab Premium](https://about.gitlab.com/pricing/) 11.8.
......
......@@ -12,7 +12,7 @@ module Gitlab
include ::Gitlab::Ci::Config::Entry::Processable
ALLOWED_WHEN = %w[on_success on_failure always manual].freeze
ALLOWED_KEYS = %i[trigger].freeze
ALLOWED_KEYS = %i[trigger parallel].freeze
validations do
validates :config, allowed_keys: ALLOWED_KEYS + PROCESSABLE_ALLOWED_KEYS
......@@ -48,7 +48,12 @@ module Gitlab
inherit: false,
metadata: { allowed_needs: %i[job bridge] }
attributes :when, :allow_failure
entry :parallel, Entry::Product::Parallel,
description: 'Parallel configuration for this job.',
inherit: false,
metadata: { allowed_strategies: %i(matrix) }
attributes :when, :allow_failure, :parallel
def self.matching?(name, config)
!name.to_s.start_with?('.') &&
......@@ -66,7 +71,8 @@ module Gitlab
needs: (needs_value if needs_defined?),
ignore: ignored?,
when: self.when,
scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage
scheduling_type: needs_defined? && !bridge_needs ? :dag : :stage,
parallel: has_parallel? ? parallel_value : nil
).compact
end
......
......@@ -22,6 +22,13 @@ module Gitlab
greater_than_or_equal_to: 2,
less_than_or_equal_to: Entry::Product::Parallel::PARALLEL_LIMIT },
allow_nil: true
validate do
next unless opt(:allowed_strategies)
next if opt(:allowed_strategies).include?(:numeric)
errors.add(:config, 'cannot use "parallel: <number>".')
end
end
def value
......@@ -38,6 +45,13 @@ module Gitlab
validations do
validates :config, allowed_keys: PERMITTED_KEYS
validates :config, required_keys: PERMITTED_KEYS
validate do
next unless opt(:allowed_strategies)
next if opt(:allowed_strategies).include?(:matrix)
errors.add(:config, 'cannot use "parallel: matrix".')
end
end
entry :matrix, Entry::Product::Matrix,
......
......@@ -244,6 +244,52 @@ RSpec.describe Gitlab::Ci::Config::Entry::Bridge do
end
end
end
context 'when bridge config contains parallel' do
let(:config) { { trigger: 'some/project', parallel: parallel_config } }
context 'when parallel config is a number' do
let(:parallel_config) { 2 }
describe '#valid?' do
it { is_expected.not_to be_valid }
end
describe '#errors' do
it 'returns an error message' do
expect(subject.errors)
.to include(/cannot use "parallel: <number>"/)
end
end
end
context 'when parallel config is a matrix' do
let(:parallel_config) do
{ matrix: [{ PROVIDER: 'aws', STACK: %w[monitoring app1] },
{ PROVIDER: 'gcp', STACK: %w[data] }] }
end
describe '#valid?' do
it { is_expected.to be_valid }
end
describe '#value' do
it 'is returns a bridge job configuration' do
expect(subject.value).to eq(
name: :my_bridge,
trigger: { project: 'some/project' },
ignore: false,
stage: 'test',
only: { refs: %w[branches tags] },
parallel: { matrix: [{ 'PROVIDER' => ['aws'], 'STACK' => %w(monitoring app1) },
{ 'PROVIDER' => ['gcp'], 'STACK' => %w(data) }] },
variables: {},
scheduling_type: :stage
)
end
end
end
end
end
describe '#manual_action?' do
......
......@@ -4,9 +4,10 @@ require 'fast_spec_helper'
require_dependency 'active_model'
RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do
subject(:parallel) { described_class.new(config) }
let(:metadata) { {} }
subject(:parallel) { described_class.new(config, **metadata) }
context 'with invalid config' do
shared_examples 'invalid config' do |error_message|
describe '#valid?' do
it { is_expected.not_to be_valid }
......@@ -19,6 +20,7 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do
end
end
context 'with invalid config' do
context 'when it is not a numeric value' do
let(:config) { true }
......@@ -63,6 +65,12 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do
expect(parallel.value).to match(number: config)
end
end
context 'when :numeric is not allowed' do
let(:metadata) { { allowed_strategies: [:matrix] } }
it_behaves_like 'invalid config', /cannot use "parallel: <number>"/
end
end
end
......@@ -89,6 +97,12 @@ RSpec.describe ::Gitlab::Ci::Config::Entry::Product::Parallel do
])
end
end
context 'when :matrix is not allowed' do
let(:metadata) { { allowed_strategies: [:numeric] } }
it_behaves_like 'invalid config', /cannot use "parallel: matrix"/
end
end
end
end
# frozen_string_literal: true
require 'spec_helper'
RSpec.describe Ci::CreatePipelineService do
let_it_be(:project) { create(:project, :repository) }
let_it_be(:user) { project.owner }
let(:service) { described_class.new(project, user, { ref: 'master' }) }
let(:pipeline) { service.execute(:push) }
before do
stub_ci_pipeline_yaml_file(config)
end
context 'job:parallel' do
context 'numeric' do
let(:config) do
<<-EOY
job:
script: "echo job"
parallel: 3
EOY
end
it 'creates the pipeline' do
expect(pipeline).to be_created_successfully
end
it 'creates 3 jobs' do
expect(pipeline.processables.pluck(:name)).to contain_exactly(
'job 1/3', 'job 2/3', 'job 3/3'
)
end
end
context 'matrix' do
let(:config) do
<<-EOY
job:
script: "echo job"
parallel:
matrix:
- PROVIDER: ovh
STACK: [monitoring, app]
- PROVIDER: [gcp, vultr]
STACK: [data]
EOY
end
it 'creates the pipeline' do
expect(pipeline).to be_created_successfully
end
it 'creates 4 builds with the corresponding matrix variables' do
expect(pipeline.processables.pluck(:name)).to contain_exactly(
'job: [gcp, data]', 'job: [ovh, app]', 'job: [ovh, monitoring]', 'job: [vultr, data]'
)
job1 = find_job('job: [gcp, data]')
job2 = find_job('job: [ovh, app]')
job3 = find_job('job: [ovh, monitoring]')
job4 = find_job('job: [vultr, data]')
expect(job1.scoped_variables.to_hash).to include('PROVIDER' => 'gcp', 'STACK' => 'data')
expect(job2.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'app')
expect(job3.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'monitoring')
expect(job4.scoped_variables.to_hash).to include('PROVIDER' => 'vultr', 'STACK' => 'data')
end
context 'when a bridge is using parallel:matrix' do
let(:config) do
<<-EOY
job:
stage: test
script: "echo job"
deploy:
stage: deploy
trigger:
include: child.yml
parallel:
matrix:
- PROVIDER: ovh
STACK: [monitoring, app]
- PROVIDER: [gcp, vultr]
STACK: [data]
EOY
end
it 'creates the pipeline' do
expect(pipeline).to be_created_successfully
end
it 'creates 1 build and 4 bridges with the corresponding matrix variables' do
expect(pipeline.processables.pluck(:name)).to contain_exactly(
'job', 'deploy: [gcp, data]', 'deploy: [ovh, app]', 'deploy: [ovh, monitoring]', 'deploy: [vultr, data]'
)
bridge1 = find_job('deploy: [gcp, data]')
bridge2 = find_job('deploy: [ovh, app]')
bridge3 = find_job('deploy: [ovh, monitoring]')
bridge4 = find_job('deploy: [vultr, data]')
expect(bridge1.scoped_variables.to_hash).to include('PROVIDER' => 'gcp', 'STACK' => 'data')
expect(bridge2.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'app')
expect(bridge3.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'monitoring')
expect(bridge4.scoped_variables.to_hash).to include('PROVIDER' => 'vultr', 'STACK' => 'data')
end
end
end
end
private
def find_job(name)
pipeline.processables.find { |job| job.name == name }
end
end
# frozen_string_literal: true
RSpec.shared_examples 'Pipeline Processing Service' do
let(:user) { create(:user) }
let(:project) { create(:project) }
let(:project) { create(:project, :repository) }
let(:user) { project.owner }
let(:pipeline) do
create(:ci_empty_pipeline, ref: 'master', project: project)
end
before do
stub_ci_pipeline_to_return_yaml_file
stub_not_protect_default_branch
project.add_developer(user)
end
context 'when simple pipeline is defined' do
before do
create_build('linux', stage_idx: 0)
......@@ -852,10 +844,74 @@ RSpec.shared_examples 'Pipeline Processing Service' do
end
end
context 'when a bridge job has parallel:matrix config', :sidekiq_inline do
let(:parent_config) do
<<-EOY
test:
stage: test
script: echo test
deploy:
stage: deploy
trigger:
include: .child.yml
parallel:
matrix:
- PROVIDER: ovh
STACK: [monitoring, app]
EOY
end
let(:child_config) do
<<-EOY
test:
stage: test
script: echo test
EOY
end
let(:pipeline) do
Ci::CreatePipelineService.new(project, user, { ref: 'master' }).execute(:push)
end
before do
allow_next_instance_of(Repository) do |repository|
allow(repository)
.to receive(:blob_data_at)
.with(an_instance_of(String), '.gitlab-ci.yml')
.and_return(parent_config)
allow(repository)
.to receive(:blob_data_at)
.with(an_instance_of(String), '.child.yml')
.and_return(child_config)
end
end
it 'creates pipeline with bridges, then passes the matrix variables to downstream jobs' do
expect(all_builds_names).to contain_exactly('test', 'deploy: [ovh, monitoring]', 'deploy: [ovh, app]')
expect(all_builds_statuses).to contain_exactly('pending', 'created', 'created')
succeed_pending
# bridge jobs directly transition to success
expect(all_builds_statuses).to contain_exactly('success', 'success', 'success')
bridge1 = all_builds.find_by(name: 'deploy: [ovh, monitoring]')
bridge2 = all_builds.find_by(name: 'deploy: [ovh, app]')
downstream_job1 = bridge1.downstream_pipeline.processables.first
downstream_job2 = bridge2.downstream_pipeline.processables.first
expect(downstream_job1.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'monitoring')
expect(downstream_job2.scoped_variables.to_hash).to include('PROVIDER' => 'ovh', 'STACK' => 'app')
end
end
private
def all_builds
pipeline.builds.order(:stage_idx, :id)
pipeline.processables.order(:stage_idx, :id)
end
def builds
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment